Automated Helm Test

.github/workflows/cd-post-release-tests.yml

@@ -27,6 +27,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
 
@@ -121,3 +131,59 @@ jobs:
       - name: Check Syft version
         run: |
           python -c "import syft; print(syft.__version__)"
+
+  notebook-test-k8s-k3d:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install K3d
+        run: |
+          K3D_VERSION=v5.6.0
+          wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=$K3D_VERSION bash
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install tox
+        run: |
+          pip install tox
+
+      - name: Run K8s tests
+        env:
+          SYFT_VERSION: ${{ inputs.syft_version }}
+        run: |
+          tox -e syft.test.helm

notebooks/api/0.8/10-container-images.ipynb

@@ -942,7 +942,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert job.job_worker_id is not None"
+    "# Disabling it due to Race Condition Error\n",
+    "# assert job.job_worker_id is not None"
    ]
   },
   {

notebooks/api/0.8/11-container-images-k8s.ipynb

@@ -28,6 +28,9 @@
     "# syft absolute\n",
     "import syft as sy\n",
     "\n",
+    "from getpass import getpass  # noqa\n",
+    "\n",
+    "\n",
     "sy.requires(SYFT_VERSION)\n",
     "\n",
     "# syft absolute\n",
@@ -210,11 +213,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registry = \"k3d-registry.localhost:5800\"\n",
+    "registry = os.getenv(\"SYFT_BASE_IMAGE_REGISTRY\", \"docker.io\")\n",
     "repo = \"openmined/grid-backend\"\n",
     "\n",
-    "res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n",
-    "tag = res.json()[\"tags\"][0]"
+    "if \"k3d\" in registry:\n",
+    "    res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n",
+    "    tag = res.json()[\"tags\"][0]\n",
+    "else:\n",
+    "    tag = sy.__version__"
    ]
   },
   {
@@ -322,7 +328,23 @@
    "id": "91a66871",
    "metadata": {},
    "source": [
-    "#### Add k3d  Registry in Syft"
+    "#### Add External Registry in Syft"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "32a323ca-8293-408a-a878-a954df55d787",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "external_registry = os.getenv(\"EXTERNAL_REGISTRY\", registry)\n",
+    "external_registry_username = os.getenv(\"EXTERNAL_REGISTRY_USERNAME\", None)\n",
+    "external_registry_password = os.getenv(\"EXTERNAL_REGISTRY_PASSWORD\", None)\n",
+    "\n",
+    "# external_registry = input()\n",
+    "# external_registry_username = getpass(\"Enter Registry Username\")\n",
+    "# external_registry_password = getpass(\"Enter Registry Password\")"
    ]
   },
   {
@@ -332,7 +354,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "registry_add_result = domain_client.api.services.image_registry.add(registry)\n",
+    "registry_add_result = domain_client.api.services.image_registry.add(external_registry)\n",
     "registry_add_result"
    ]
   },
@@ -488,7 +510,11 @@
    "outputs": [],
    "source": [
     "push_result = None\n",
-    "push_result = domain_client.api.services.worker_image.push(workerimage.id)\n",
+    "push_result = domain_client.api.services.worker_image.push(\n",
+    "    workerimage.id,\n",
+    "    username=external_registry_username,\n",
+    "    password=external_registry_password,\n",
+    ")\n",
     "push_result"
    ]
   },
@@ -544,6 +570,8 @@
     "    name=worker_pool_name,\n",
     "    image_uid=workerimage.id,\n",
     "    num_workers=3,\n",
+    "    reg_username=external_registry_username,\n",
+    "    reg_password=external_registry_password,\n",
     ")"
    ]
   },
@@ -1027,7 +1055,11 @@
    "source": [
     "# Push OpenDP Image to registry\n",
     "push_result = None\n",
-    "push_result = domain_client.api.services.worker_image.push(workerimage_opendp.id)\n",
+    "push_result = domain_client.api.services.worker_image.push(\n",
+    "    workerimage_opendp.id,\n",
+    "    username=external_registry_username,\n",
+    "    password=external_registry_password,\n",
+    ")\n",
     "assert isinstance(push_result, sy.SyftSuccess), str(push_result)"
    ]
   },
@@ -1040,9 +1072,7 @@
    "source": [
     "pool_name_opendp = \"opendp-pool\"\n",
     "pool_create_request = domain_client.api.services.worker_pool.pool_creation_request(\n",
-    "    pool_name=pool_name_opendp,\n",
-    "    num_workers=3,\n",
-    "    image_uid=workerimage_opendp.id,\n",
+    "    pool_name=pool_name_opendp, num_workers=3, image_uid=workerimage_opendp.id\n",
     ")\n",
     "pool_create_request"
    ]
@@ -1066,7 +1096,9 @@
    "outputs": [],
    "source": [
     "# get the pending request and approve it\n",
-    "req_result = pool_create_request.approve()\n",
+    "req_result = pool_create_request.approve(\n",
+    "    reg_username=external_registry_username, reg_password=external_registry_password\n",
+    ")\n",
     "req_result"
    ]
   },
@@ -1210,7 +1242,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "req_result = pool_image_create_request.approve()\n",
+    "req_result = pool_image_create_request.approve(\n",
+    "    reg_username=external_registry_username, reg_password=external_registry_password\n",
+    ")\n",
     "req_result"
    ]
   },

tox.ini

@@ -676,6 +676,7 @@ setenv =
     NODE_PORT = {env:NODE_PORT:9082}
     GITHUB_CI = {env:GITHUB_CI:false}
     PYTEST_MODULES = {env:PYTEST_MODULES:frontend container_workload}
+    SYFT_BASE_IMAGE_REGISTRY = {env:SYFT_BASE_IMAGE_REGISTRY:k3d-registry.localhost:5800}
 commands =
     bash -c "echo Running with GITHUB_CI=$GITHUB_CI; date"
     python -c 'import syft as sy; sy.stage_protocol_changes()'
@@ -851,77 +852,67 @@ commands =
 
 [testenv:syft.test.helm]
 description = Test Helm Chart for Kubernetes
-deps =
-    {[testenv:syft]deps}
-    {[testenv:hagrid]deps}
-    nbmake
 changedir = {toxinidir}
-passenv=HOME, USER
+passenv=HOME,USER,EXTERNAL_REGISTRY_USERNAME,EXTERNAL_REGISTRY_PASSWORD
 allowlist_externals =
-    devspace
-    kubectl
     grep
     sleep
     bash
-    kubectx
-    k3d
-    echo
-    rm
-    helm
+    tox
 setenv =
     ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
     NODE_PORT = {env:NODE_PORT:8080}
+    NODE_URL = {env:NODE_URL:http://localhost}
+    EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:not 10-container-images.ipynb}
+    SYFT_VERSION = {env:SYFT_VERSION:local}
+    EXTERNAL_REGISTRY = {env:EXTERNAL_REGISTRY:k3d-registry.localhost:5800}
 commands =
-    k3d version
+    bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE NODE_PORT=$NODE_PORT NODE_URL=$NODE_URL \
+    Excluding notebooks: $EXCLUDE_NOTEBOOKS SYFT_VERSION=$SYFT_VERSION \
+    EXTERNAL_REGISTRY=$EXTERNAL_REGISTRY; date"
+
 
+    bash -c "k3d version"
+
+    # Remvoing old clusters and volumes and registry
     ; bash -c "docker rm $(docker ps -aq) --force || true"
     bash -c "k3d cluster delete syft || true"
     bash -c "docker volume rm k3d-syft-images --force || true"
-
     bash -c "k3d registry delete k3d-registry.localhost || true"
 
+    # Creating registry and cluster
     bash -c 'k3d registry create registry.localhost --port 5800  -v `pwd`/k3d-registry:/var/lib/registry || true'
-
-    bash -c 'NODE_NAME=syft NODE_PORT=8080 && \
+    bash -c 'NODE_NAME=syft NODE_PORT=${NODE_PORT} && \
         k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
         k3d cluster start syft'
+    tox -e dev.k8s.patch.coredns
+    sleep 10
+    bash -c "kubectl --context k3d-syft create namespace syft || true"
+
+    # if syft version is local, then install local helm charts
+    # else install the helm charts from the openmined gh-pages branch
+    bash -c 'if [[ $SYFT_VERSION == "local" ]]; then \
+        echo "Installing local helm charts"; \
+        bash -c "cd packages/grid/helm && helm install --kube-context k3d-syft --namespace syft syft ./syft --set configuration.devmode=true"; \
+    else \
+        echo "Installing helm charts from repo for syft version: ${SYFT_VERSION}"; \
+        bash -c "helm repo add openmined https://openmined.github.io/PySyft/helm && helm repo update openmined"; \
+        bash -c "helm install --kube-context k3d-syft --namespace syft syft openmined/syft --version=${SYFT_VERSION} --set configuration.devmode=true"; \
+    fi'
 
-    ; skopeo list-tags --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend
-    ; skopeo inspect --tls-verify=false docker://k3d-registry.localhost:5800/openmined/grid-backend:f1725f
-    ; helm uninstall --kube-context k3d-syft --namespace syft syft
-    ; helm install --kube-context k3d-syft --namespace syft syft ./syft
-    ; k3d cluster create syft -p "8080:80@loadbalancer" && k3d cluster start syft
-
-    sleep 50
-
-    bash -c 'cd packages/grid && \
-        kubectl --context k3d-syft create namespace syft || true; \
-        helm install --kube-context k3d-syft --namespace syft syft ./helm/syft --debug'
-
+    ; wait for everything else to be loaded
     bash packages/grid/scripts/wait_for.sh service frontend --context k3d-syft --namespace syft
     bash -c '(kubectl logs service/frontend --context k3d-syft --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
-
-    ; wait for everything else to be loaded
     bash packages/grid/scripts/wait_for.sh service mongo --context k3d-syft --namespace syft
     bash packages/grid/scripts/wait_for.sh service backend --context k3d-syft --namespace syft
     bash packages/grid/scripts/wait_for.sh service proxy --context k3d-syft --namespace syft
-
     bash -c '(kubectl logs service/backend --context k3d-syft --namespace syft -f &) | grep -q "Application startup complete" || true'
 
 
-    ; frontend
-    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
-        echo "Starting frontend"; date; \
-        pytest tests/integration -m frontend -p no:randomly -k "test_serves_domain_frontend" --co; \
-        pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no -k "test_serves_domain_frontend"; \
-        return=$?; \
-        echo "Finished frontend"; date; \
-        exit $return; \
-    fi'
-
-    ; ignore 06 because of opendp on arm64
-    pytest --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv
+    # Run Notebook tests
+    tox -e e2e.test.notebook
 
+    # Cleanup
     bash -c "k3d cluster delete syft || true"
     bash -c "docker volume rm k3d-syft-images --force || true"
 
@@ -1114,32 +1105,37 @@ commands =
 # The below is the notebook test suite for point at external servers
 [testenv:e2e.test.notebook]
 description = E2E Notebook tests
-changedir = {toxinidir}/notebooks
+changedir = {toxinidir}
 allowlist_externals =
     bash
+passenv = EXTERNAL_REGISTRY,EXTERNAL_REGISTRY_USERNAME,EXTERNAL_REGISTRY_PASSWORD
 setenv =
     ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
     NODE_PORT = {env:NODE_PORT:8080}
     NODE_URL = {env:NODE_URL:http://localhost}
     EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:}
-    SYFT_VERSION = {env:SYFT_VERSION:latest}
+    SYFT_VERSION = {env:SYFT_VERSION:local}
 commands =
     bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE NODE_PORT=$NODE_PORT NODE_URL=$NODE_URL \
-    Excluding notebooks: $EXCLUDE_NOTEBOOKS SYFT_VERSION=$SYFT_VERSION; date"
+    Excluding notebooks: $EXCLUDE_NOTEBOOKS SYFT_VERSION=$SYFT_VERSION \
+    EXTERNAL_REGISTRY=$EXTERNAL_REGISTRY; date"
 
 
     # Schema for EXLUDE_NOTEBOOKS is
     # for excluding
     # notebook1.ipynb, notebook2.ipynb
     # EXCLUDE_NOTEBOOKS=not notebook1.ipynb and not notebook2.ipynb
     bash -c "pip install pytest pytest-randomly nbmake"
-    #If the syft version is latest install the latest version of syft
+
+    # If the syft version is local install the local version
     # else install the version of syft specified
-    bash -c "if [[ $SYFT_VERSION == 'latest' ]]; then \
-        pip install syft[data_science]; \
+    bash -c " if [[ $SYFT_VERSION == 'local' ]]; then \
+        echo 'Building local syft'; \
+        pip install packages/syft[data_science]; \
     else \
-        pip install syft[data_science]==$SYFT_VERSION; \
+        echo 'Installing syft version: ${SYFT_VERSION}'; \
+        pip install syft[data_science]==${SYFT_VERSION}; \
     fi"
 
 
-    pytest api/0.8 --nbmake -p no:randomly -vvvv --nbmake-timeout=1000 -k '{env:EXCLUDE_NOTEBOOKS:}'
+    pytest notebooks/api/0.8 --nbmake -p no:randomly -vvvv --nbmake-timeout=1000 -k '{env:EXCLUDE_NOTEBOOKS:}'