From ec857e677012181bae3843de5705bd74b9e1ca1e Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 13 Sep 2023 21:26:54 +0530 Subject: [PATCH 01/24] [ci] add `dev` K8s deployments --- .github/workflows/cd-syft-dev.yml | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .github/workflows/cd-syft-dev.yml diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml new file mode 100644 index 00000000000..9213532d3c4 --- /dev/null +++ b/.github/workflows/cd-syft-dev.yml @@ -0,0 +1,123 @@ +name: CD - Syft - Deploy `dev` to K8s + +on: + schedule: + - cron: "0 */3 * * *" + + workflow_dispatch: + inputs: + none: + description: "Run Tests Manually" + required: false + +jobs: + deploy-syft: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Azure container registry + uses: azure/docker-login@v1 + with: + login-server: ${{ secrets.ACR_SERVER }} + username: ${{ secrets.ACR_USERNAME }} + password: ${{ secrets.ACR_PASSWORD }} + + - name: Set Grid package version + id: grid + shell: bash + run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT + + - name: Build and push `grid-backend` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages + file: ./packages/grid/backend/backend.dockerfile + push: true + platforms: linux/amd64 + target: backend + tags: | + ${{ secrets.ACR_SERVER }}/grid-backend:dev + ${{ secrets.ACR_SERVER }}/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-backend:${{ github.sha }} + # cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} + # cache-to: type=inline + + - name: Build and push `grid-frontend` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages/grid/frontend + file: ./packages/grid/frontend/frontend.dockerfile + push: true + platforms: linux/amd64 + tags: | + ${{ secrets.ACR_SERVER }}/grid-frontend:dev + ${{ secrets.ACR_SERVER }}/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-frontend:${{ github.sha }} + target: grid-ui-development + # cache-from: type=registry,ref=${{ steps.meta_grid_frontend.outputs.tags }} + # cache-to: type=inline + + - name: Build and push `grid-enclave` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages + file: ./packages/grid/worker/worker.dockerfile + push: true + platforms: linux/amd64 + target: worker + tags: | + ${{ secrets.ACR_SERVER }}/grid-enclave:dev + ${{ secrets.ACR_SERVER }}/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-enclave:${{ github.sha }} + # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} + # cache-to: type=inline + + - name: Build and push `grid-headscale` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages/grid/vpn + file: ./packages/grid/vpn/headscale.dockerfile + push: true + platforms: linux/amd64 + tags: | + ${{ secrets.ACR_SERVER }}/grid-headscale:dev + ${{ secrets.ACR_SERVER }}/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-headscale:${{ github.sha }} + # cache-from: type=registry,ref=${{ steps.meta_grid_headscale.outputs.tags }} + # cache-to: type=inline + + - name: Build and push `grid-tailscale` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages/grid/vpn + file: ./packages/grid/vpn/tailscale.dockerfile + push: true + platforms: linux/amd64 + tags: | + ${{ secrets.ACR_SERVER }}/grid-tailscale:dev + ${{ secrets.ACR_SERVER }}/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-tailscale:${{ github.sha }} + # cache-from: type=registry,ref=${{ steps.meta_grid_tailscale.outputs.tags }} + # cache-to: type=inline + + - name: Build and push `grid-vpn-iptables` image to DockerHub + uses: docker/build-push-action@v2 + with: + context: ./packages/grid/vpn + file: ./packages/grid/vpn/iptables.dockerfile + push: true + platforms: linux/amd64 + tags: | + ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:dev + ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:${{ github.sha }} + # cache-from: type=registry,ref=${{ steps.meta_grid_vpn_iptables.outputs.tags }} + # cache-to: type=inline From ced33516887782fe068bbc4a36c79c641df497cb Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 13 Sep 2023 21:47:20 +0530 Subject: [PATCH 02/24] [ci] setup python 3.11 for cd-syft-dev --- .github/workflows/cd-syft-dev.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 9213532d3c4..5ef4cb2cd27 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -16,6 +16,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 97848efaafea1fdbb86243132026d5b78a076318 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 13 Sep 2023 23:42:22 +0530 Subject: [PATCH 03/24] [ci] build & upload dev helm charts --- .github/workflows/cd-syft-dev.yml | 81 +++++++++++++++++++++++-------- tox.ini | 6 ++- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 5ef4cb2cd27..ecad67f6321 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -1,8 +1,9 @@ name: CD - Syft - Deploy `dev` to K8s on: - schedule: - - cron: "0 */3 * * *" + # TODO: Re-enable once merged & tested + # schedule: + # - cron: "0 */3 * * *" workflow_dispatch: inputs: @@ -11,11 +12,17 @@ on: required: false jobs: - deploy-syft: + deploy-syft-dev: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Checkout Openmined/infrastructure + uses: actions/checkout@v4 + with: + repository: OpenMined/infrastructure + path: infrastructure + - name: Set up Python uses: actions/setup-python@v3 with: @@ -49,9 +56,9 @@ jobs: platforms: linux/amd64 target: backend tags: | - ${{ secrets.ACR_SERVER }}/grid-backend:dev - ${{ secrets.ACR_SERVER }}/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-backend:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} # cache-to: type=inline @@ -63,9 +70,9 @@ jobs: push: true platforms: linux/amd64 tags: | - ${{ secrets.ACR_SERVER }}/grid-frontend:dev - ${{ secrets.ACR_SERVER }}/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-frontend:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ github.sha }} target: grid-ui-development # cache-from: type=registry,ref=${{ steps.meta_grid_frontend.outputs.tags }} # cache-to: type=inline @@ -79,9 +86,9 @@ jobs: platforms: linux/amd64 target: worker tags: | - ${{ secrets.ACR_SERVER }}/grid-enclave:dev - ${{ secrets.ACR_SERVER }}/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-enclave:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} # cache-to: type=inline @@ -93,9 +100,9 @@ jobs: push: true platforms: linux/amd64 tags: | - ${{ secrets.ACR_SERVER }}/grid-headscale:dev - ${{ secrets.ACR_SERVER }}/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-headscale:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_headscale.outputs.tags }} # cache-to: type=inline @@ -107,9 +114,9 @@ jobs: push: true platforms: linux/amd64 tags: | - ${{ secrets.ACR_SERVER }}/grid-tailscale:dev - ${{ secrets.ACR_SERVER }}/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-tailscale:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_tailscale.outputs.tags }} # cache-to: type=inline @@ -121,8 +128,40 @@ jobs: push: true platforms: linux/amd64 tags: | - ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:dev - ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/grid-vpn-iptables:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev + ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} + ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_vpn_iptables.outputs.tags }} # cache-to: type=inline + + - name: Build Helm Chart & Copy to infra + shell: bash + run: | + # install k3d + wget https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64 + mv k3d-linux-amd64 k3d + chmod +x k3d + export PATH=`pwd`:$PATH + k3d version + DEVSPACE_VERSION=v6.3.2 + + # Install devspace + curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace + chmod +x devspace + devspace version + + export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }}/ + tox -e syft.build.helm + + rm -rf infrastructure/gitops/environments/dev/ + mkdir -p infrastructure/gitops/environments/dev/ + cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/ + + - name: Commit & push changes to Infra Repo + uses: EndBug/add-and-commit@v9 + with: + author_name: Madhava Jay + author_email: madhava@openmined.org + message: "[env] Update dev helm charts" + cwd: infrastructure + pull: NO-PULL diff --git a/tox.ini b/tox.ini index d0ed8ec06d1..fc9e74dd385 100644 --- a/tox.ini +++ b/tox.ini @@ -738,7 +738,7 @@ description = Build Helm Chart for Kubernetes deps = pyyaml changedir = {toxinidir} -passenv=HOME, USER +passenv=HOME, USER, CONTAINER_REGISTRY allowlist_externals = devspace kubectl @@ -763,7 +763,9 @@ commands = bash -c 'rm packages/grid/helm/syft/templates/* || true' bash -c 'cd packages/grid && \ - devspace deploy --skip-build --no-warn --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py' + [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \ + echo $REGISTRY_FLAG && \ + devspace deploy --skip-build --no-warn ${REGISTRY_FLAG} --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py' bash -c 'cd packages/grid/helm && \ helm lint syft' From 55820cc7d52dc7ef7601c8c518bc7e2433a57523 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 14 Sep 2023 00:17:18 +0530 Subject: [PATCH 04/24] [ci] remove debug comment --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index fc9e74dd385..f3bc508ea06 100644 --- a/tox.ini +++ b/tox.ini @@ -764,7 +764,6 @@ commands = bash -c 'cd packages/grid && \ [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \ - echo $REGISTRY_FLAG && \ devspace deploy --skip-build --no-warn ${REGISTRY_FLAG} --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py' bash -c 'cd packages/grid/helm && \ From d95fa299ddbf0b9145441a43999edc41ecbda75c Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 14 Sep 2023 12:25:30 +0530 Subject: [PATCH 05/24] remove qemu --- .github/workflows/cd-syft-dev.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index ecad67f6321..84a96cb0a8e 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -28,9 +28,6 @@ jobs: with: python-version: "3.11" - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -53,7 +50,6 @@ jobs: context: ./packages file: ./packages/grid/backend/backend.dockerfile push: true - platforms: linux/amd64 target: backend tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev @@ -68,7 +64,6 @@ jobs: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile push: true - platforms: linux/amd64 tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} @@ -83,7 +78,6 @@ jobs: context: ./packages file: ./packages/grid/worker/worker.dockerfile push: true - platforms: linux/amd64 target: worker tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev @@ -98,7 +92,6 @@ jobs: context: ./packages/grid/vpn file: ./packages/grid/vpn/headscale.dockerfile push: true - platforms: linux/amd64 tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} @@ -112,7 +105,6 @@ jobs: context: ./packages/grid/vpn file: ./packages/grid/vpn/tailscale.dockerfile push: true - platforms: linux/amd64 tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} @@ -126,7 +118,6 @@ jobs: context: ./packages/grid/vpn file: ./packages/grid/vpn/iptables.dockerfile push: true - platforms: linux/amd64 tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} @@ -143,9 +134,9 @@ jobs: chmod +x k3d export PATH=`pwd`:$PATH k3d version - DEVSPACE_VERSION=v6.3.2 # Install devspace + DEVSPACE_VERSION=v6.3.2 curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace chmod +x devspace devspace version From 97f163bf8c74860c58b52ce43d632a0fda9e6b48 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 14 Sep 2023 12:37:34 +0530 Subject: [PATCH 06/24] update devspace version --- .github/workflows/cd-syft-dev.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 84a96cb0a8e..655640944de 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -128,15 +128,17 @@ jobs: - name: Build Helm Chart & Copy to infra shell: bash run: | + K3D_VERSION=v5.6.0 + DEVSPACE_VERSION=v6.3.3 + # install k3d - wget https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64 + wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 mv k3d-linux-amd64 k3d chmod +x k3d export PATH=`pwd`:$PATH k3d version # Install devspace - DEVSPACE_VERSION=v6.3.2 curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace chmod +x devspace devspace version From de9193936ef4114f0e0b7105eb0fa3ff622ac70e Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 14 Sep 2023 15:24:51 +0530 Subject: [PATCH 07/24] checkout infra git with token --- .github/workflows/cd-syft-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 655640944de..a7ac07bb32e 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -22,6 +22,7 @@ jobs: with: repository: OpenMined/infrastructure path: infrastructure + token: ${{ secrets.SYFT_BUMP_TOKEN }} - name: Set up Python uses: actions/setup-python@v3 From 00d8d46dad935ad66f60a9367b046da167a77f0b Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 19 Sep 2023 10:18:17 +0530 Subject: [PATCH 08/24] [ci] use alfred the bot --- .github/workflows/cd-syft-dev.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index a7ac07bb32e..c85213c9074 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -22,7 +22,7 @@ jobs: with: repository: OpenMined/infrastructure path: infrastructure - token: ${{ secrets.SYFT_BUMP_TOKEN }} + token: ${{ secrets.OM_BOT_TOKEN }} - name: Set up Python uses: actions/setup-python@v3 @@ -154,8 +154,8 @@ jobs: - name: Commit & push changes to Infra Repo uses: EndBug/add-and-commit@v9 with: - author_name: Madhava Jay - author_email: madhava@openmined.org + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[env] Update dev helm charts" cwd: infrastructure pull: NO-PULL From 46e482e63e74d48a0aefd27c38f8c86ff08cd08e Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 19 Sep 2023 10:20:34 +0530 Subject: [PATCH 09/24] [ci] disable enclave in dev k8s --- .github/workflows/cd-syft-dev.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index c85213c9074..cd268503e71 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -73,19 +73,20 @@ jobs: # cache-from: type=registry,ref=${{ steps.meta_grid_frontend.outputs.tags }} # cache-to: type=inline - - name: Build and push `grid-enclave` image to DockerHub - uses: docker/build-push-action@v4 - with: - context: ./packages - file: ./packages/grid/worker/worker.dockerfile - push: true - target: worker - tags: | - ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ github.sha }} - # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} - # cache-to: type=inline + # TODO: Re-enable once we have Enclave up and running + # - name: Build and push `grid-enclave` image to DockerHub + # uses: docker/build-push-action@v4 + # with: + # context: ./packages + # file: ./packages/grid/worker/worker.dockerfile + # push: true + # target: worker + # tags: | + # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev + # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} + # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ github.sha }} + # # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} + # # cache-to: type=inline - name: Build and push `grid-headscale` image to DockerHub uses: docker/build-push-action@v4 From 92edddc7105bf6f9c40a24515d7cd30fd78df4f8 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:51:19 +0530 Subject: [PATCH 10/24] modified to self hosted runners --- .github/workflows/cd-syft-dev.yml | 15 +++++++++------ .github/workflows/cd-syft.yml | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index cd268503e71..6bf5a6d7949 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -13,16 +13,18 @@ on: jobs: deploy-syft-dev: - runs-on: ubuntu-latest + runs-on: om-ci-16vcpu-ubuntu2204 steps: - uses: actions/checkout@v4 - - name: Checkout Openmined/infrastructure + # Checkout Infra repo (nested) + - name: Checkout Infra Repo uses: actions/checkout@v4 with: - repository: OpenMined/infrastructure - path: infrastructure + repository: ${{ secrets.INFRA_REPO }} + ref: "main" token: ${{ secrets.OM_BOT_TOKEN }} + path: infrastructure - name: Set up Python uses: actions/setup-python@v3 @@ -158,5 +160,6 @@ jobs: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "[env] Update dev helm charts" - cwd: infrastructure - pull: NO-PULL + add: "." + push: "origin main" + cwd: "./infrastructure/" diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 3fbc3a5394b..7e6347baec9 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -157,7 +157,7 @@ jobs: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} message: "Update K8s Manifests from Syft Repo" - add: "*" + add: "." push: "origin main" cwd: "./infrastructure/" From 58e2f19ea700e465634fcfbc617cb5a887bfbcbf Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:19:42 +0530 Subject: [PATCH 11/24] Added version flag in builds --- .github/workflows/cd-syft-dev.yml | 16 ++++++---------- tox.ini | 5 +++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 6bf5a6d7949..bda0e7a207c 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -56,8 +56,7 @@ jobs: target: backend tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} # cache-to: type=inline @@ -69,8 +68,7 @@ jobs: push: true tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }} target: grid-ui-development # cache-from: type=registry,ref=${{ steps.meta_grid_frontend.outputs.tags }} # cache-to: type=inline @@ -98,8 +96,7 @@ jobs: push: true tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_headscale.outputs.tags }} # cache-to: type=inline @@ -111,8 +108,7 @@ jobs: push: true tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_tailscale.outputs.tags }} # cache-to: type=inline @@ -124,8 +120,7 @@ jobs: push: true tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev - ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} - ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }} # cache-from: type=registry,ref=${{ steps.meta_grid_vpn_iptables.outputs.tags }} # cache-to: type=inline @@ -148,6 +143,7 @@ jobs: devspace version export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }}/ + export VERSION=dev-${{github.sha}} tox -e syft.build.helm rm -rf infrastructure/gitops/environments/dev/ diff --git a/tox.ini b/tox.ini index f3bc508ea06..ecfd40bfb44 100644 --- a/tox.ini +++ b/tox.ini @@ -738,7 +738,7 @@ description = Build Helm Chart for Kubernetes deps = pyyaml changedir = {toxinidir} -passenv=HOME, USER, CONTAINER_REGISTRY +passenv=HOME, USER, CONTAINER_REGISTRY , VERSION allowlist_externals = devspace kubectl @@ -764,7 +764,8 @@ commands = bash -c 'cd packages/grid && \ [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \ - devspace deploy --skip-build --no-warn ${REGISTRY_FLAG} --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py' + [[ -n "$VERSION" ]] && VERSION_FLAG="--var VERSION=$VERSION" || VERSION_FLAG="" && \ + devspace deploy --skip-build --no-warn ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" --render 2>/dev/null | python3 helm/helm.py' bash -c 'cd packages/grid/helm && \ helm lint syft' From a07ba0dcf5fc873e8853bae6789e4410c5030e1f Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:24:08 +0530 Subject: [PATCH 12/24] removed slash at url endpoints of container registry --- .github/workflows/cd-syft-dev.yml | 2 +- packages/grid/devspace.yaml | 18 +++++++++--------- tox.ini | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index bda0e7a207c..69c92144c7e 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -142,7 +142,7 @@ jobs: chmod +x devspace devspace version - export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }}/ + export CONTAINER_REGISTRY=${{ secrets.ACR_SERVER }} export VERSION=dev-${{github.sha}} tox -e syft.build.helm diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 6bfebc6669b..5b150d2203c 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -20,14 +20,14 @@ pipelines: vars: DEVSPACE_ENV_FILE: "default.env" - CONTAINER_REGISTRY: "docker.io/" + CONTAINER_REGISTRY: "docker.io" VERSION: "0.8.2-beta.30" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible images: backend: - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_BACKEND}" + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}" buildKit: {} dockerfile: ./backend/backend.dockerfile context: ../ @@ -35,7 +35,7 @@ images: - dev-latest - "${VERSION}" tailscale: - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_TAILSCALE}" + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_TAILSCALE}" buildKit: {} dockerfile: ./vpn/tailscale.dockerfile context: ./vpn @@ -43,7 +43,7 @@ images: - dev-latest - "${VERSION}" frontend: - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_FRONTEND}" + image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}" buildKit: args: ["--target", "grid-ui-production"] dockerfile: ./frontend/frontend.dockerfile @@ -83,7 +83,7 @@ deployments: env: - name: SERVICE_NAME value: "proxy" - # - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_TAILSCALE}:${VERSION}" + # - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_TAILSCALE}:${VERSION}" # securityContext: # capabilities: # add: ["NET_ADMIN", "SYS_MODULE"] @@ -123,7 +123,7 @@ deployments: version: "0.9.0" values: containers: - - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_BACKEND}:${VERSION}" + - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" volumeMounts: - containerPath: /storage volume: @@ -228,7 +228,7 @@ deployments: version: "0.9.0" values: containers: - - image: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_FRONTEND}:${VERSION}" + - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}:${VERSION}" env: - name: VERSION value: "${VERSION}" @@ -249,7 +249,7 @@ dev: ports: - port: "27017" backend: - imageSelector: "${CONTAINER_REGISTRY}${DOCKER_IMAGE_BACKEND}" + imageSelector: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}" env: - name: DEV_MODE value: "True" @@ -261,4 +261,4 @@ dev: commands: dev: command: |- - devspace --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ dev --config=devspace.yaml + devspace --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 dev --config=devspace.yaml diff --git a/tox.ini b/tox.ini index ecfd40bfb44..bf4d310d705 100644 --- a/tox.ini +++ b/tox.ini @@ -624,7 +624,7 @@ commands = # --var DOMAIN_NAME=$NODE_NAME \ # --var NETWORK_CHECK_INTERVAL=5 \ # --var TEST_MODE=1 \ - # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \ + # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ # build -b' # bash -c 'NODE_NAME=test-gateway-1 NODE_PORT=9081 && \ @@ -635,7 +635,7 @@ commands = # --var NETWORK_CHECK_INTERVAL=5 \ # --var ASSOCIATION_TIMEOUT=100 \ # --var TEST_MODE=1 \ - # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \ + # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ # deploy -b -p gateway; \ # do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -651,7 +651,7 @@ commands = --var DOMAIN_CHECK_INTERVAL=5 \ --var ASSOCIATION_TIMEOUT=100 \ --var TEST_MODE=1 \ - --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \ + --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ deploy -b; \ do ((--r))||exit;echo "retrying" && sleep 20;done)' @@ -667,7 +667,7 @@ commands = # --var DOMAIN_CHECK_INTERVAL=5 \ # --var ASSOCIATION_TIMEOUT=100 \ # --var TEST_MODE=1 \ - # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \ + # --var CONTAINER_REGISTRY=k3d-registry.localhost:12345 \ # deploy -b -p domain; \ # do ((--r))||exit;echo "retrying" && sleep 20;done)' From bacb8d593c3d735b808fb3573f8aec4116eb1777 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:27:40 +0530 Subject: [PATCH 13/24] removed unused comments --- .github/workflows/cd-syft-dev.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 69c92144c7e..1a4f922369e 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -70,8 +70,6 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }} target: grid-ui-development - # cache-from: type=registry,ref=${{ steps.meta_grid_frontend.outputs.tags }} - # cache-to: type=inline # TODO: Re-enable once we have Enclave up and running # - name: Build and push `grid-enclave` image to DockerHub @@ -83,10 +81,7 @@ jobs: # target: worker # tags: | # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev - # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} - # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ github.sha }} - # # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} - # # cache-to: type=inline + # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }} - name: Build and push `grid-headscale` image to DockerHub uses: docker/build-push-action@v4 @@ -97,8 +92,6 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }} - # cache-from: type=registry,ref=${{ steps.meta_grid_headscale.outputs.tags }} - # cache-to: type=inline - name: Build and push `grid-tailscale` image to DockerHub uses: docker/build-push-action@v4 @@ -109,8 +102,6 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }} - # cache-from: type=registry,ref=${{ steps.meta_grid_tailscale.outputs.tags }} - # cache-to: type=inline - name: Build and push `grid-vpn-iptables` image to DockerHub uses: docker/build-push-action@v2 @@ -121,8 +112,6 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }} - # cache-from: type=registry,ref=${{ steps.meta_grid_vpn_iptables.outputs.tags }} - # cache-to: type=inline - name: Build Helm Chart & Copy to infra shell: bash From 14f1d5ace9e75f483a38659434bc96710ab64117 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:28:33 +0530 Subject: [PATCH 14/24] removed some more unused comments --- .github/workflows/cd-syft-dev.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 1a4f922369e..02af22ebfb4 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -57,8 +57,6 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }} - # cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} - # cache-to: type=inline - name: Build and push `grid-frontend` image to DockerHub uses: docker/build-push-action@v4 From 213b16ceab772653066a2fe85bb94e357b08cb27 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Wed, 20 Sep 2023 11:12:09 +0000 Subject: [PATCH 15/24] bump version --- packages/hagrid/.bumpversion.cfg | 2 +- packages/hagrid/hagrid/manifest_template.yml | 4 ++-- packages/hagrid/hagrid/version.py | 2 +- packages/hagrid/setup.py | 2 +- scripts/hagrid_hash | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg index 9d38890f4f1..b04a6814d13 100644 --- a/packages/hagrid/.bumpversion.cfg +++ b/packages/hagrid/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.71 +current_version = 0.3.72 tag = False tag_name = {new_version} commit = True diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 12730acbeba..0b517425623 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 -hagrid_version: 0.3.71 +hagrid_version: 0.3.72 syft_version: 0.8.2-beta.30 dockerTag: 0.8.2-beta.30 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: 8daa30a460b679585f4f6d0b9707bfc0110ca27a +hash: f0d04c5e0a71295271f2fda2647a7bb9a0affe19 target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py index 5c26289f637..12026873d77 100644 --- a/packages/hagrid/hagrid/version.py +++ b/packages/hagrid/hagrid/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # HAGrid Version -__version__ = "0.3.71" +__version__ = "0.3.72" if __name__ == "__main__": print(__version__) diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py index 22eb365fdf6..49272db197b 100644 --- a/packages/hagrid/setup.py +++ b/packages/hagrid/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup -__version__ = "0.3.71" +__version__ = "0.3.72" DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]} diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash index ba276c38159..effd467bb4e 100644 --- a/scripts/hagrid_hash +++ b/scripts/hagrid_hash @@ -1 +1 @@ -e32f2c4364e17407a56a8cf9fb828d5e +8cfc7ab679fc789a602f5d694f35b37f From c6f930b92f57eb76f4706b5f1ef7b0f164d44abd Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Thu, 21 Sep 2023 14:10:57 +1000 Subject: [PATCH 16/24] Make free space on Ubuntu runner which builds docker images - Update Docker Compose binary --- .github/workflows/pr-tests-frontend.yml | 2 +- .github/workflows/pr-tests-stack-arm64.yml | 2 +- .github/workflows/pr-tests-stack-public.yml | 2 +- .github/workflows/pr-tests-stack.yml | 4 ++-- .github/workflows/pr-tests-syft.yml | 10 +++++++++- packages/hagrid/hagrid/deps.py | 4 ++-- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml index d813983dc86..4a215141480 100644 --- a/.github/workflows/pr-tests-frontend.yml +++ b/.github/workflows/pr-tests-frontend.yml @@ -148,7 +148,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose diff --git a/.github/workflows/pr-tests-stack-arm64.yml b/.github/workflows/pr-tests-stack-arm64.yml index 09df1420680..d29185523cc 100644 --- a/.github/workflows/pr-tests-stack-arm64.yml +++ b/.github/workflows/pr-tests-stack-arm64.yml @@ -70,7 +70,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose diff --git a/.github/workflows/pr-tests-stack-public.yml b/.github/workflows/pr-tests-stack-public.yml index 6658e470b45..ea33c1f11bb 100644 --- a/.github/workflows/pr-tests-stack-public.yml +++ b/.github/workflows/pr-tests-stack-public.yml @@ -111,7 +111,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml index 72e0003145d..b449aef57fa 100644 --- a/.github/workflows/pr-tests-stack.yml +++ b/.github/workflows/pr-tests-stack.yml @@ -123,7 +123,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose @@ -334,7 +334,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 77a2ceeca6c..82e9a2b87f7 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -208,6 +208,14 @@ jobs: # run: | # sudo chown -R $USER:$USER $HOME - uses: actions/checkout@v3 + + # free 10GB of space + - name: Remove unnecessary files + if: matrix.os == 'ubuntu-latest' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Check for file changes uses: dorny/paths-filter@v2 id: changes @@ -253,7 +261,7 @@ jobs: shell: bash run: | mkdir -p ~/.docker/cli-plugins - DOCKER_COMPOSE_VERSION=v2.16.0 + DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose docker compose version diff --git a/packages/hagrid/hagrid/deps.py b/packages/hagrid/hagrid/deps.py index a4340faec5f..143e309e65b 100644 --- a/packages/hagrid/hagrid/deps.py +++ b/packages/hagrid/hagrid/deps.py @@ -48,7 +48,7 @@ You are running an old version of docker, possibly on Linux. You need to install v2. At the time of writing this, if you are on linux you need to run the following: -DOCKER_COMPOSE_VERSION=v2.16.0 +DOCKER_COMPOSE_VERSION=v2.21.0 curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 \ -o ~/.docker/cli-plugins/docker-compose chmod +x ~/.docker/cli-plugins/docker-compose @@ -770,7 +770,7 @@ def check_syft( "windows": "choco install docker-desktop -y", "linux": ( "mkdir -p ~/.docker/cli-plugins\n" - + "DOCKER_COMPOSE_VERSION=v2.16.0\n" + + "DOCKER_COMPOSE_VERSION=v2.21.0\n" + "curl -sSL https://github.com/docker/compose/releases/download/" + "${DOCKER_COMPOSE_VERSION}/docker-compose-linux-x86_64 " + "-o ~/.docker/cli-plugins/docker-compose\n" From 44e0878dc338d7cc8cf370ffc3052f5a05f672c5 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Thu, 21 Sep 2023 14:39:26 +1000 Subject: [PATCH 17/24] Fix scanning - clear space for backend build - use correct seaweedfs docker repo image name --- .github/workflows/container-scan.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml index 78d87c8fa49..803478053d9 100644 --- a/.github/workflows/container-scan.yml +++ b/.github/workflows/container-scan.yml @@ -19,6 +19,13 @@ jobs: steps: - uses: actions/checkout@v3 + # free 10GB of space + - name: Remove unnecessary files + if: matrix.os == 'ubuntu-latest' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # Build the docker image for testing - name: Build a Docker image run: DOCKER_BUILDKIT=1 docker build -f packages/grid/backend/backend.dockerfile packages -t backend:${{ github.sha }} --no-cache @@ -56,6 +63,13 @@ jobs: # This is where you will need to introduce the Snyk API token created with your Snyk account SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + # free 10GB of space + - name: Remove unnecessary files + if: matrix.os == 'ubuntu-latest' + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # Build the docker image for testing - name: Build a Docker image shell: bash @@ -524,7 +538,7 @@ jobs: continue-on-error: true uses: aquasecurity/trivy-action@master with: - image-ref: "seaweedfs:3.55" + image-ref: "chrislusf/seaweedfs:3.55" format: "cyclonedx" output: "seaweedfs-trivy-results.sbom.json" timeout: "10m0s" @@ -541,7 +555,7 @@ jobs: continue-on-error: true uses: aquasecurity/trivy-action@master with: - image-ref: "seaweedfs:3.55" + image-ref: "chrislusf/seaweedfs:3.55" format: "sarif" output: "trivy-results.sarif" timeout: "10m0s" From 6fab4657bce3202ac53e1a5e80f0247ad39d07ce Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Thu, 21 Sep 2023 14:57:53 +1000 Subject: [PATCH 18/24] Fix issue where clear space wasn't running --- .github/workflows/container-scan.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml index 803478053d9..a3e09f2aacd 100644 --- a/.github/workflows/container-scan.yml +++ b/.github/workflows/container-scan.yml @@ -21,7 +21,6 @@ jobs: # free 10GB of space - name: Remove unnecessary files - if: matrix.os == 'ubuntu-latest' run: | sudo rm -rf /usr/share/dotnet sudo rm -rf "$AGENT_TOOLSDIRECTORY" @@ -65,7 +64,6 @@ jobs: # free 10GB of space - name: Remove unnecessary files - if: matrix.os == 'ubuntu-latest' run: | sudo rm -rf /usr/share/dotnet sudo rm -rf "$AGENT_TOOLSDIRECTORY" From 0ea0bd0f46a627ad2e3f3a41847984aaac7e35d9 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:34:52 +0530 Subject: [PATCH 19/24] upgraded to new personal grained access token in pysyft --- .github/workflows/cd-syft-dev.yml | 2 +- .github/workflows/cd-syft.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 02af22ebfb4..bd9afa8c250 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -23,7 +23,7 @@ jobs: with: repository: ${{ secrets.INFRA_REPO }} ref: "main" - token: ${{ secrets.OM_BOT_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} path: infrastructure - name: Set up Python diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 7e6347baec9..582b44b6ca7 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -48,7 +48,7 @@ jobs: - uses: actions/checkout@v3 with: - token: ${{ secrets.OM_BOT_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} - name: Set up Python uses: actions/setup-python@v3 @@ -137,7 +137,7 @@ jobs: with: repository: ${{ secrets.INFRA_REPO }} ref: "main" - token: ${{ secrets.OM_BOT_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} path: infrastructure # This step will copy the generated K8s manifest files to the correct directory in Infra repo From c0b96705587776b272a7e0069d0baf412e10f972 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:39:19 +0530 Subject: [PATCH 20/24] updated hagrid token --- .github/workflows/cd-hagrid.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml index f1f44444c12..94bf7509f83 100644 --- a/.github/workflows/cd-hagrid.yml +++ b/.github/workflows/cd-hagrid.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.SYFT_BUMP_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} - name: Install checksumdir run: | pip install --upgrade checksumdir @@ -85,13 +85,12 @@ jobs: - name: Commit changes if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}} - uses: EndBug/add-and-commit@v7 + uses: EndBug/add-and-commit@v9 with: - author_name: Madhava Jay - author_email: madhava@openmined.org - message: "bump version" + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} + message: "[hagrid] bump version" add: "['./packages/hagrid/.bumpversion.cfg','./packages/hagrid/setup.py','./packages/hagrid/hagrid/version.py', './scripts/hagrid_hash', './packages/hagrid/hagrid/manifest_template.yml']" - pull_strategy: NO-PULL - name: Build and publish if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}} From d140cbdf2e88654680fc5fd41b239295d72bc931 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:48:01 +0530 Subject: [PATCH 21/24] removed unsued tokens updated bot token in syft cli and notebooks cleanup PR --- .github/workflows/cd-syft.yml | 2 +- .github/workflows/cd-syftcli.yml | 17 ++++++----------- .../workflows/post-merge-cleanup-notebooks.yml | 10 ++++++---- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 582b44b6ca7..e5b1eea3b94 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -90,7 +90,7 @@ jobs: with: author_name: ${{ secrets.OM_BOT_NAME }} author_email: ${{ secrets.OM_BOT_EMAIL }} - message: "bump version" + message: "[syft]bump version" add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml', 'packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml']" - name: Scheduled Build and Publish diff --git a/.github/workflows/cd-syftcli.yml b/.github/workflows/cd-syftcli.yml index a8d7ac9eb22..63287f1532b 100644 --- a/.github/workflows/cd-syftcli.yml +++ b/.github/workflows/cd-syftcli.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.SYFT_BUMP_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} - name: Install checksumdir run: | @@ -83,15 +83,14 @@ jobs: if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }} run: echo $(checksumdir packages/syftcli/syftcli -e version.py) > ./scripts/syftcli_hash - - name: Commit changes + - name: Commit changes to Syft CLI if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }} - uses: EndBug/add-and-commit@v7 + uses: EndBug/add-and-commit@v9 with: - author_name: Madhava Jay - author_email: madhava@openmined.org - message: "bump version" + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} + message: "[syftcli]bump version" add: "['./packages/syftcli/.bumpversion.cfg','./packages/syftcli/setup.py','./packages/syftcli/syftcli/version.py', './scripts/syftcli_hash']" - pull_strategy: NO-PULL - name: Build and publish if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }} @@ -114,8 +113,6 @@ jobs: steps: - uses: actions/checkout@v3 - with: - token: ${{ secrets.SYFT_BUMP_TOKEN }} - name: Set up Python uses: actions/setup-python@v3 @@ -152,8 +149,6 @@ jobs: steps: - uses: actions/checkout@v3 - with: - token: ${{ secrets.SYFT_BUMP_TOKEN }} - name: Deployed Version run: echo ${{needs.deploy-syft-cli.outputs.deployed_version}} diff --git a/.github/workflows/post-merge-cleanup-notebooks.yml b/.github/workflows/post-merge-cleanup-notebooks.yml index b25d73e86ca..8b7f07ae4c3 100644 --- a/.github/workflows/post-merge-cleanup-notebooks.yml +++ b/.github/workflows/post-merge-cleanup-notebooks.yml @@ -15,16 +15,18 @@ jobs: steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.SYFT_BUMP_TOKEN }} + token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Commit changes + + - name: Commit changes to remove notebooks uses: EndBug/add-and-commit@v9 with: - author_name: Madhava Jay - author_email: madhava@openmined.org + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} message: "cleanup notebooks" remove: "-r notebooks/Experimental/" commit: "-a" From 5acda79ea625c1e58372f437016d79cd9ca6e560 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:54:13 +0530 Subject: [PATCH 22/24] updated infra token --- .github/workflows/cd-syft-dev.yml | 2 +- .github/workflows/cd-syft.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index bd9afa8c250..cee1b8e4e28 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -23,7 +23,7 @@ jobs: with: repository: ${{ secrets.INFRA_REPO }} ref: "main" - token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} + token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }} path: infrastructure - name: Set up Python diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index e5b1eea3b94..506c7a9d351 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -137,7 +137,7 @@ jobs: with: repository: ${{ secrets.INFRA_REPO }} ref: "main" - token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} + token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }} path: infrastructure # This step will copy the generated K8s manifest files to the correct directory in Infra repo From 59650e654082e6d2c57d250e1f79b43700d34b7b Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:57:30 +0530 Subject: [PATCH 23/24] added a sample notebook to check cleanup --- notebooks/Experimental/test.ipynb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 notebooks/Experimental/test.ipynb diff --git a/notebooks/Experimental/test.ipynb b/notebooks/Experimental/test.ipynb new file mode 100644 index 00000000000..e69de29bb2d From da5935ddb1a7378afa43ee7f7184c684a1bbe934 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 21 Sep 2023 13:05:01 +0530 Subject: [PATCH 24/24] Added permission to home directory --- .github/workflows/cd-syft-dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index cee1b8e4e28..65fc02e9093 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -15,6 +15,10 @@ jobs: deploy-syft-dev: runs-on: om-ci-16vcpu-ubuntu2204 steps: + - name: Permission to home directory + run: | + sudo chown -R $USER:$USER $HOME + - uses: actions/checkout@v4 # Checkout Infra repo (nested)