Merge branch 'dev' into container_health

.github/workflows/cd-syft-dev.yml

@@ -84,7 +84,7 @@ jobs:
           echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
       - name: Build and push `syft` image to registry
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages
           file: ./packages/grid/syft-client/syft.Dockerfile
@@ -95,7 +95,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/syft-client:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-backend` image to registry
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages
           file: ./packages/grid/backend/backend.dockerfile
@@ -107,7 +107,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-frontend` image to registry
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/frontend
           file: ./packages/grid/frontend/frontend.dockerfile
@@ -119,7 +119,7 @@ jobs:
           target: grid-ui-development
 
       - name: Build and push `grid-seaweedfs` image to registry
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/seaweedfs
           file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
@@ -130,7 +130,7 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build and push `grid-enclave-attestation` image to registry
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/enclave/attestation
           file: ./packages/grid/enclave/attestation/attestation.dockerfile

.github/workflows/cd-syft.yml

@@ -185,7 +185,7 @@ jobs:
 
       - name: Build and push `grid-backend` image to DockerHub
         id: grid-backend-build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages
           file: ./packages/grid/backend/backend.dockerfile
@@ -203,7 +203,7 @@ jobs:
 
       - name: Build and push `grid-frontend` image to DockerHub
         id: grid-frontend-build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/frontend
           file: ./packages/grid/frontend/frontend.dockerfile
@@ -221,7 +221,7 @@ jobs:
 
       - name: Build and push `grid-seaweedfs` image to DockerHub
         id: grid-seaweedfs-build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/seaweedfs
           file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
@@ -241,7 +241,7 @@ jobs:
       - name: Build and push `grid-enclave-attestation` image to DockerHub
         if: ${{ endsWith(matrix.runner, '-x64') }}
         id: grid-enclave-attestation-build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/grid/enclave/attestation
           file: ./packages/grid/enclave/attestation/attestation.dockerfile
@@ -259,7 +259,7 @@ jobs:
 
       - name: Build and push `syft` image to registry
         id: syft-build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: ./packages/
           file: ./packages/grid/syft-client/syft.Dockerfile

notebooks/api/0.8/11-container-images-k8s.ipynb

@@ -246,12 +246,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# syft absolute\n",
+    "from syft.util.util import get_latest_tag\n",
+    "\n",
     "registry = os.getenv(\"SYFT_BASE_IMAGE_REGISTRY\", \"docker.io\")\n",
     "repo = \"openmined/grid-backend\"\n",
     "\n",
     "if \"k3d\" in registry:\n",
-    "    res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n",
-    "    tag = res.json()[\"tags\"][0]\n",
+    "    tag = get_latest_tag(registry, repo)\n",
     "else:\n",
     "    tag = sy.__version__"
    ]

packages/grid/helm/syft/templates/backend/backend-statefulset.yaml

@@ -109,7 +109,14 @@ spec:
             - name: SMTP_USERNAME
               value: {{ .Values.node.smtp.username | quote }}
             - name: SMTP_PASSWORD
+              {{- if .Values.node.smtp.existingSecret }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.node.smtp.existingSecret }}
+                  key: smtpPassword
+              {{ else }}
               value: {{ .Values.node.smtp.password | quote }}
+              {{ end }}
             - name: EMAIL_SENDER
               value: {{ .Values.node.smtp.from | quote}}
             # SeaweedFS

packages/grid/helm/syft/values.yaml

@@ -175,6 +175,8 @@ node:
 
   # SMTP Settings
   smtp:
+    # Existing secret for SMTP with key 'smtpPassword'
+    existingSecret: null
     host: smtp.sendgrid.net
     port: 587
     from: [email protected]
@@ -195,7 +197,7 @@ node:
   resourcesPreset: xlarge
   resources: null
 
-  # Seaweed secret name. Override this if you want to use a self-managed secret.
+  # Backend secret name. Override this if you want to use a self-managed secret.
   # Secret must contain the following keys:
   # - defaultRootPassword
   secretKeyName: backend-secret

packages/syft/src/syft/util/util.py

@@ -8,9 +8,11 @@
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from copy import deepcopy
+from datetime import datetime
 import functools
 import hashlib
 from itertools import repeat
+import json
 import multiprocessing
 import multiprocessing as mp
 from multiprocessing import set_start_method
@@ -959,3 +961,33 @@ def sanitize_html(html: str) -> str:
         clean_content_tags=policy["remove"],
         attributes=attributes,
     )
+
+
+def parse_iso8601_date(date_string: str) -> datetime:
+    # Handle variable length of microseconds by trimming to 6 digits
+    if "." in date_string:
+        base_date, microseconds = date_string.split(".")
+        microseconds = microseconds.rstrip("Z")  # Remove trailing 'Z'
+        microseconds = microseconds[:6]  # Trim to 6 digits
+        date_string = f"{base_date}.{microseconds}Z"
+    return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def get_latest_tag(registry: str, repo: str) -> str | None:
+    repo_url = f"http://{registry}/v2/{repo}"
+    res = requests.get(url=f"{repo_url}/tags/list", timeout=5)
+    tags = res.json().get("tags", [])
+
+    tag_times = []
+    for tag in tags:
+        manifest_response = requests.get(f"{repo_url}/manifests/{tag}", timeout=5)
+        manifest = manifest_response.json()
+        created_time = json.loads(manifest["history"][0]["v1Compatibility"])["created"]
+        created_datetime = parse_iso8601_date(created_time)
+        tag_times.append((tag, created_datetime))
+
+    # sort tags by datetime
+    tag_times.sort(key=lambda x: x[1], reverse=True)
+    if len(tag_times) > 0:
+        return tag_times[0][0]
+    return None

tests/integration/container_workload/pool_image_test.py

@@ -5,7 +5,6 @@
 # third party
 import numpy as np
 import pytest
-import requests
 
 # syft absolute
 import syft as sy
@@ -19,13 +18,13 @@
 from syft.service.worker.worker_pool import SyftWorker
 from syft.service.worker.worker_pool import WorkerPool
 from syft.types.uid import UID
+from syft.util.util import get_latest_tag
 
 registry = os.getenv("SYFT_BASE_IMAGE_REGISTRY", "docker.io")
 repo = "openmined/grid-backend"
 
 if "k3d" in registry:
-    res = requests.get(url=f"http://{registry}/v2/{repo}/tags/list")
-    tag = res.json()["tags"][0]
+    tag = get_latest_tag(registry, repo)
 else:
     tag = sy.__version__