diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 7ab86f9d89d..00dd4acb816 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -84,7 +84,7 @@ jobs: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - name: Build and push `syft` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/syft-client/syft.Dockerfile @@ -95,7 +95,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/syft-client:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-backend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -107,7 +107,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-frontend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -119,7 +119,7 @@ jobs: target: grid-ui-development - name: Build and push `grid-seaweedfs` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -130,7 +130,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-enclave-attestation` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 5bb32744f4b..41b68a4e357 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -185,7 +185,7 @@ jobs: - name: Build and push `grid-backend` image to DockerHub id: grid-backend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -203,7 +203,7 @@ jobs: - name: Build and push `grid-frontend` image to DockerHub id: grid-frontend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -221,7 +221,7 @@ jobs: - name: Build and push `grid-seaweedfs` image to DockerHub id: grid-seaweedfs-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -241,7 +241,7 @@ jobs: - name: Build and push `grid-enclave-attestation` image to DockerHub if: ${{ endsWith(matrix.runner, '-x64') }} id: grid-enclave-attestation-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile @@ -259,7 +259,7 @@ jobs: - name: Build and push `syft` image to registry id: syft-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/ file: ./packages/grid/syft-client/syft.Dockerfile diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 27340a8d9e5..59bf6b5da9e 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -246,12 +246,14 @@ "metadata": {}, "outputs": [], "source": [ + "# syft absolute\n", + "from syft.util.util import get_latest_tag\n", + "\n", "registry = os.getenv(\"SYFT_BASE_IMAGE_REGISTRY\", \"docker.io\")\n", "repo = \"openmined/grid-backend\"\n", "\n", "if \"k3d\" in registry:\n", - " res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n", - " tag = res.json()[\"tags\"][0]\n", + " tag = get_latest_tag(registry, repo)\n", "else:\n", " tag = sy.__version__" ] diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 106d2fee893..be0a35d6245 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -109,7 +109,14 @@ spec: - name: SMTP_USERNAME value: {{ .Values.node.smtp.username | quote }} - name: SMTP_PASSWORD + {{- if .Values.node.smtp.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.node.smtp.existingSecret }} + key: smtpPassword + {{ else }} value: {{ .Values.node.smtp.password | quote }} + {{ end }} - name: EMAIL_SENDER value: {{ .Values.node.smtp.from | quote}} # SeaweedFS diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 2644eac26e4..631475b2462 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -175,6 +175,8 @@ node: # SMTP Settings smtp: + # Existing secret for SMTP with key 'smtpPassword' + existingSecret: null host: smtp.sendgrid.net port: 587 from: noreply@openmined.org @@ -195,7 +197,7 @@ node: resourcesPreset: xlarge resources: null - # Seaweed secret name. Override this if you want to use a self-managed secret. + # Backend secret name. Override this if you want to use a self-managed secret. # Secret must contain the following keys: # - defaultRootPassword secretKeyName: backend-secret diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index 0860eb9e5e4..3689f319073 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -8,9 +8,11 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from copy import deepcopy +from datetime import datetime import functools import hashlib from itertools import repeat +import json import multiprocessing import multiprocessing as mp from multiprocessing import set_start_method @@ -959,3 +961,33 @@ def sanitize_html(html: str) -> str: clean_content_tags=policy["remove"], attributes=attributes, ) + + +def parse_iso8601_date(date_string: str) -> datetime: + # Handle variable length of microseconds by trimming to 6 digits + if "." in date_string: + base_date, microseconds = date_string.split(".") + microseconds = microseconds.rstrip("Z") # Remove trailing 'Z' + microseconds = microseconds[:6] # Trim to 6 digits + date_string = f"{base_date}.{microseconds}Z" + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ") + + +def get_latest_tag(registry: str, repo: str) -> str | None: + repo_url = f"http://{registry}/v2/{repo}" + res = requests.get(url=f"{repo_url}/tags/list", timeout=5) + tags = res.json().get("tags", []) + + tag_times = [] + for tag in tags: + manifest_response = requests.get(f"{repo_url}/manifests/{tag}", timeout=5) + manifest = manifest_response.json() + created_time = json.loads(manifest["history"][0]["v1Compatibility"])["created"] + created_datetime = parse_iso8601_date(created_time) + tag_times.append((tag, created_datetime)) + + # sort tags by datetime + tag_times.sort(key=lambda x: x[1], reverse=True) + if len(tag_times) > 0: + return tag_times[0][0] + return None diff --git a/tests/integration/container_workload/pool_image_test.py b/tests/integration/container_workload/pool_image_test.py index a3a53aa2385..bb84e5883aa 100644 --- a/tests/integration/container_workload/pool_image_test.py +++ b/tests/integration/container_workload/pool_image_test.py @@ -5,7 +5,6 @@ # third party import numpy as np import pytest -import requests # syft absolute import syft as sy @@ -19,13 +18,13 @@ from syft.service.worker.worker_pool import SyftWorker from syft.service.worker.worker_pool import WorkerPool from syft.types.uid import UID +from syft.util.util import get_latest_tag registry = os.getenv("SYFT_BASE_IMAGE_REGISTRY", "docker.io") repo = "openmined/grid-backend" if "k3d" in registry: - res = requests.get(url=f"http://{registry}/v2/{repo}/tags/list") - tag = res.json()["tags"][0] + tag = get_latest_tag(registry, repo) else: tag = sy.__version__