From 417adf112e033338351ee69077a94d9f69a376ae Mon Sep 17 00:00:00 2001 From: Spencer Koch Date: Thu, 13 Jun 2024 19:38:49 -0500 Subject: [PATCH 1/4] add existingSecret check for SMTP password --- .../helm/syft/templates/backend/backend-statefulset.yaml | 7 +++++++ packages/grid/helm/syft/values.yaml | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 106d2fee893..be0a35d6245 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -109,7 +109,14 @@ spec: - name: SMTP_USERNAME value: {{ .Values.node.smtp.username | quote }} - name: SMTP_PASSWORD + {{- if .Values.node.smtp.existingSecret }} + valueFrom: + secretKeyRef: + name: {{ .Values.node.smtp.existingSecret }} + key: smtpPassword + {{ else }} value: {{ .Values.node.smtp.password | quote }} + {{ end }} - name: EMAIL_SENDER value: {{ .Values.node.smtp.from | quote}} # SeaweedFS diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 2644eac26e4..631475b2462 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -175,6 +175,8 @@ node: # SMTP Settings smtp: + # Existing secret for SMTP with key 'smtpPassword' + existingSecret: null host: smtp.sendgrid.net port: 587 from: noreply@openmined.org @@ -195,7 +197,7 @@ node: resourcesPreset: xlarge resources: null - # Seaweed secret name. Override this if you want to use a self-managed secret. + # Backend secret name. Override this if you want to use a self-managed secret. # Secret must contain the following keys: # - defaultRootPassword secretKeyName: backend-secret From 67a7e6fdb3ee6d73ea2a377e1f253bbd07afd2b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jun 2024 02:12:42 +0000 Subject: [PATCH 2/4] build(deps): bump docker/build-push-action from 5 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cd-syft-dev.yml | 10 +++++----- .github/workflows/cd-syft.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 7ab86f9d89d..00dd4acb816 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -84,7 +84,7 @@ jobs: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - name: Build and push `syft` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/syft-client/syft.Dockerfile @@ -95,7 +95,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/syft-client:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-backend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -107,7 +107,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-frontend` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -119,7 +119,7 @@ jobs: target: grid-ui-development - name: Build and push `grid-seaweedfs` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -130,7 +130,7 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-enclave-attestation` image to registry - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 5bb32744f4b..41b68a4e357 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -185,7 +185,7 @@ jobs: - name: Build and push `grid-backend` image to DockerHub id: grid-backend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -203,7 +203,7 @@ jobs: - name: Build and push `grid-frontend` image to DockerHub id: grid-frontend-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -221,7 +221,7 @@ jobs: - name: Build and push `grid-seaweedfs` image to DockerHub id: grid-seaweedfs-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/seaweedfs file: ./packages/grid/seaweedfs/seaweedfs.dockerfile @@ -241,7 +241,7 @@ jobs: - name: Build and push `grid-enclave-attestation` image to DockerHub if: ${{ endsWith(matrix.runner, '-x64') }} id: grid-enclave-attestation-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/grid/enclave/attestation file: ./packages/grid/enclave/attestation/attestation.dockerfile @@ -259,7 +259,7 @@ jobs: - name: Build and push `syft` image to registry id: syft-build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: ./packages/ file: ./packages/grid/syft-client/syft.Dockerfile From 82eda43218f8794e6073153c4dd1fd723b4ea650 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Mon, 24 Jun 2024 15:43:02 +1000 Subject: [PATCH 3/4] Fixed issue where tags in local repo are not returned in create order --- .../api/0.8/11-container-images-k8s.ipynb | 6 ++-- packages/syft/src/syft/util/util.py | 32 +++++++++++++++++++ .../container_workload/pool_image_test.py | 5 ++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 27340a8d9e5..59bf6b5da9e 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -246,12 +246,14 @@ "metadata": {}, "outputs": [], "source": [ + "# syft absolute\n", + "from syft.util.util import get_latest_tag\n", + "\n", "registry = os.getenv(\"SYFT_BASE_IMAGE_REGISTRY\", \"docker.io\")\n", "repo = \"openmined/grid-backend\"\n", "\n", "if \"k3d\" in registry:\n", - " res = requests.get(url=f\"http://{registry}/v2/{repo}/tags/list\")\n", - " tag = res.json()[\"tags\"][0]\n", + " tag = get_latest_tag(registry, repo)\n", "else:\n", " tag = sy.__version__" ] diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index 0860eb9e5e4..d34db89f365 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -8,9 +8,11 @@ from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from copy import deepcopy +from datetime import datetime import functools import hashlib from itertools import repeat +import json import multiprocessing import multiprocessing as mp from multiprocessing import set_start_method @@ -959,3 +961,33 @@ def sanitize_html(html: str) -> str: clean_content_tags=policy["remove"], attributes=attributes, ) + + +def parse_iso8601_date(date_string: str) -> datetime: + # Handle variable length of microseconds by trimming to 6 digits + if "." in date_string: + base_date, microseconds = date_string.split(".") + microseconds = microseconds.rstrip("Z") # Remove trailing 'Z' + microseconds = microseconds[:6] # Trim to 6 digits + date_string = f"{base_date}.{microseconds}Z" + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%fZ") + + +def get_latest_tag(registry: str, repo: str) -> str | None: + repo_url = f"http://{registry}/v2/{repo}" + res = requests.get(url=f"{repo_url}/tags/list") + tags = res.json().get("tags", []) + + tag_times = [] + for tag in tags: + manifest_response = requests.get(f"{repo_url}/manifests/{tag}") + manifest = manifest_response.json() + created_time = json.loads(manifest["history"][0]["v1Compatibility"])["created"] + created_datetime = parse_iso8601_date(created_time) + tag_times.append((tag, created_datetime)) + + # sort tags by datetime + tag_times.sort(key=lambda x: x[1], reverse=True) + if len(tag_times) > 0: + return tag_times[0][0] + return None diff --git a/tests/integration/container_workload/pool_image_test.py b/tests/integration/container_workload/pool_image_test.py index a3a53aa2385..bb84e5883aa 100644 --- a/tests/integration/container_workload/pool_image_test.py +++ b/tests/integration/container_workload/pool_image_test.py @@ -5,7 +5,6 @@ # third party import numpy as np import pytest -import requests # syft absolute import syft as sy @@ -19,13 +18,13 @@ from syft.service.worker.worker_pool import SyftWorker from syft.service.worker.worker_pool import WorkerPool from syft.types.uid import UID +from syft.util.util import get_latest_tag registry = os.getenv("SYFT_BASE_IMAGE_REGISTRY", "docker.io") repo = "openmined/grid-backend" if "k3d" in registry: - res = requests.get(url=f"http://{registry}/v2/{repo}/tags/list") - tag = res.json()["tags"][0] + tag = get_latest_tag(registry, repo) else: tag = sy.__version__ From 1e1882e5c6c6c447a0d8e172d8490dca160fc2d5 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Mon, 24 Jun 2024 15:54:47 +1000 Subject: [PATCH 4/4] Added timeout --- packages/syft/src/syft/util/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/util/util.py b/packages/syft/src/syft/util/util.py index d34db89f365..3689f319073 100644 --- a/packages/syft/src/syft/util/util.py +++ b/packages/syft/src/syft/util/util.py @@ -975,12 +975,12 @@ def parse_iso8601_date(date_string: str) -> datetime: def get_latest_tag(registry: str, repo: str) -> str | None: repo_url = f"http://{registry}/v2/{repo}" - res = requests.get(url=f"{repo_url}/tags/list") + res = requests.get(url=f"{repo_url}/tags/list", timeout=5) tags = res.json().get("tags", []) tag_times = [] for tag in tags: - manifest_response = requests.get(f"{repo_url}/manifests/{tag}") + manifest_response = requests.get(f"{repo_url}/manifests/{tag}", timeout=5) manifest = manifest_response.json() created_time = json.loads(manifest["history"][0]["v1Compatibility"])["created"] created_datetime = parse_iso8601_date(created_time)