Merge branch 'dev' into tests/add_mounted_data_test

OpenMined · Feb 1, 2024 · 608bf6e · 608bf6e
2 parents 0593baf + 901850b
commit 608bf6e
Show file tree

Hide file tree

Showing 65 changed files with 2,438 additions and 393 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.4-beta.14
+current_version = 0.8.4-beta.15
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml
@@ -42,7 +42,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml
@@ -30,7 +30,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -111,7 +111,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/pr-tests-hagrid.yml b/.github/workflows/pr-tests-hagrid.yml
@@ -35,7 +35,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/pr-tests-stack-arm64.yml b/.github/workflows/pr-tests-stack-arm64.yml
@@ -41,7 +41,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/pr-tests-stack-public.yml b/.github/workflows/pr-tests-stack-public.yml
@@ -37,7 +37,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -61,7 +61,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -251,7 +251,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -333,7 +333,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -531,7 +531,7 @@ jobs:
           sudo chown -R $USER:$USER $HOME
       - uses: actions/checkout@v4
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -544,6 +544,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Add K3d Registry
+        run: |
+          sudo python ./scripts/patch_hosts.py --add-k3d-registry
+
       # free 10GB of space
       - name: Remove unnecessary files
         if: matrix.os == 'ubuntu-latest'
@@ -616,18 +620,51 @@ jobs:
           GITHUB_CI: true
         shell: bash
         run: |
-          # install k3d
           K3D_VERSION=v5.6.0
+          DEVSPACE_VERSION=v6.3.10
+          # install k3d
           wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
           mv k3d-linux-amd64 k3d
           chmod +x k3d
           export PATH=`pwd`:$PATH
           k3d version
-          DEVSPACE_VERSION=v6.3.3
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
           tox -e stack.test.integration.k8s
           tox -e syft.build.helm
           tox -e syft.package.helm
           # tox -e syft.test.helm
+
+      - name: Get current timestamp
+        id: date
+        if: failure()
+        shell: bash
+        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT
+
+      - name: Collect logs from k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          mkdir -p ./k8s-logs
+          kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-logs-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-logs-${{ steps.date.outputs.date }}.txt
+          ls -la ./k8s-logs
+
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: steps.changes.outputs.stack == 'true' && failure()
+        with:
+          name: k8s-logs-${{ matrix.os }}-${{ steps.date.outputs.date }}
+          path: ./k8s-logs/
+
+      - name: Cleanup k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          export PATH=`pwd`:$PATH
+          k3d cluster delete testgateway1 || true
+          k3d cluster delete testdomain1 || true
+          k3d registry delete k3d-registry.localhost || true
diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml
@@ -49,7 +49,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -134,7 +134,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -214,7 +214,7 @@ jobs:
           docker system prune --all --force
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}
@@ -312,7 +312,7 @@ jobs:
       #     sudo chown -R $USER:$USER $HOME
       - uses: actions/checkout@v4
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml
@@ -31,7 +31,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Check for file changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           base: ${{ github.ref }}

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.4-beta.14"
+__version__ = "0.8.4-beta.15"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb
@@ -120,7 +120,7 @@
     "\n",
     "RUN pip install pydicom\n",
     "\n",
-    "\"\"\""
+    "\"\"\".strip()"
    ]
   },
   {
@@ -205,7 +205,7 @@
    "source": [
     "workerimage: SyftWorkerImage = None\n",
     "for image in dockerfile_list:\n",
-    "    if image.config.dockerfile == custom_dockerfile_str:\n",
+    "    if not image.is_prebuilt and image.config.dockerfile == custom_dockerfile_str:\n",
     "        workerimage = image\n",
     "        break\n",
     "\n",
@@ -1051,7 +1051,7 @@
     "FROM openmined/grid-backend:0.8.4-beta.12\n",
     "\n",
     "RUN pip install opendp\n",
-    "\"\"\"\n",
+    "\"\"\".strip()\n",
     "\n",
     "docker_config_2 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_2)"
    ]
@@ -1202,7 +1202,7 @@
     "FROM openmined/grid-backend:0.8.4-beta.12\n",
     "\n",
     "RUN pip install recordlinkage\n",
-    "\"\"\"\n",
+    "\"\"\".strip()\n",
     "\n",
     "docker_config_3 = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str_3)\n",
     "\n",
@@ -1406,7 +1406,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.2"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,