From db8840d1648220eb7b0d0fe8b2ce133383035eda Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:31:05 +0530 Subject: [PATCH 1/3] revert cd syft to original state --- .github/workflows/cd-syft.yml | 336 +++++++++++++++++----------------- 1 file changed, 168 insertions(+), 168 deletions(-) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 46034ee9ee1..b5ffa680e89 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -60,19 +60,19 @@ jobs: python -m pip install --upgrade pip pip install --upgrade tox setuptools wheel twine bump2version PyYAML - # - name: Bump the Version - # run: | - # ls **/VERSION | xargs -I {} python {} - # cat packages/grid/devspace.yaml | grep '0\.' - # bump2version prenum --allow-dirty --no-commit - # tox -e lint || true - # ls **/VERSION | xargs -I {} python {} - # cat packages/grid/devspace.yaml | grep '0\.' - # python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION) - - # - name: Check Protocol Version - # run: | - # tox -e syft.protocol.check + - name: Bump the Version + run: | + ls **/VERSION | xargs -I {} python {} + cat packages/grid/devspace.yaml | grep '0\.' + bump2version prenum --allow-dirty --no-commit + tox -e lint || true + ls **/VERSION | xargs -I {} python {} + cat packages/grid/devspace.yaml | grep '0\.' + python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION) + + - name: Check Protocol Version + run: | + tox -e syft.protocol.check - name: Build Helm Chart shell: bash @@ -98,39 +98,39 @@ jobs: tox -e syft.build.helm tox -e syft.package.helm - # - name: Check and Bump Protocol Version - # run: | - # export BUMP=True - # tox -e syft.protocol.check - - # - name: Commit changes to Syft - # uses: EndBug/add-and-commit@v9 - # with: - # author_name: ${{ secrets.OM_BOT_NAME }} - # author_email: ${{ secrets.OM_BOT_EMAIL }} - # message: "[syft]bump version" - # add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']" - - # - name: Scheduled Build and Publish - # if: github.event_name == 'schedule' - # run: | - # tox -e syft.publish - # twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* - - # - name: Manual Build and Publish - # if: github.event_name != 'schedule' - # run: | - # tox -e syft.publish - # if [ "${{ github.event.inputs.release_platform }}" = "REAL_PYPI" ]; then - # twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* - # fi - # if [ "${{ github.event.inputs.release_platform }}" = "TEST_PYPI" ]; then - # twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* - # fi - # if [ "${{ github.event.inputs.release_platform }}" = "REAL_AND_TEST_PYPI" ]; then - # twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* - # twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* - # fi + - name: Check and Bump Protocol Version + run: | + export BUMP=True + tox -e syft.protocol.check + + - name: Commit changes to Syft + uses: EndBug/add-and-commit@v9 + with: + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} + message: "[syft]bump version" + add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']" + + - name: Scheduled Build and Publish + if: github.event_name == 'schedule' + run: | + tox -e syft.publish + twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* + + - name: Manual Build and Publish + if: github.event_name != 'schedule' + run: | + tox -e syft.publish + if [ "${{ github.event.inputs.release_platform }}" = "REAL_PYPI" ]; then + twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* + fi + if [ "${{ github.event.inputs.release_platform }}" = "TEST_PYPI" ]; then + twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* + fi + if [ "${{ github.event.inputs.release_platform }}" = "REAL_AND_TEST_PYPI" ]; then + twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/* + twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* + fi - name: Set Grid package version id: grid-version @@ -149,128 +149,128 @@ jobs: echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT fi - # # Checkout Infra repo (nested) - # - name: Checkout Infra Repo - # uses: actions/checkout@v3 - # with: - # repository: ${{ secrets.INFRA_REPO }} - # ref: "main" - # token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }} - # path: infrastructure - - # # This step will copy the generated K8s manifest files to the correct directory in Infra repo - # - name: Copy files to Infra Repo - # run: | - # if $is_pre_release; then - # rm -rf infrastructure/gitops/environments/pre-release/. - # cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/ - # else - # rm -rf infrastructure/gitops/environments/stable/. - # cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/ - # fi - - # - name: Commit changes to Infra Repo - # uses: EndBug/add-and-commit@v9 - # with: - # author_name: ${{ secrets.OM_BOT_NAME }} - # author_email: ${{ secrets.OM_BOT_EMAIL }} - # message: "Update K8s Manifests from Syft Repo" - # add: "." - # push: "origin main" - # cwd: "./infrastructure/" - - # - name: Create SyftCLI Config assets - # run: | - # pip install pyyaml - # python scripts/create_syftcli_config.py - - # - name: GitHub Release - # uses: softprops/action-gh-release@v1 - # with: - # name: v${{ steps.release_checks.outputs.github_release_version }} - # generate_release_notes: true - # prerelease: ${{ steps.release_checks.outputs.is_pre_release }} - # files: | - # ./packages/syftcli/manifest.yml - # ./build/syftcli-config/* - # ./packages/hagrid/hagrid/manifest_template.yml - # tag_name: v${{ steps.release_checks.outputs.github_release_version }} - - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - - # - name: Set up Docker Buildx - # id: buildx - # uses: docker/setup-buildx-action@v2 - - # - name: Login to Docker - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKER_LOGIN }} - # password: ${{ secrets.DOCKER_PASSWORD }} - - # - name: Docker meta for Grid-Backend - # id: meta_grid_backend - # uses: docker/metadata-action@v3 - # with: - # images: openmined/grid-backend - # tags: | - # type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} - # type=raw,value=beta - - # - name: Build and push `grid-backend` image to DockerHub - # uses: docker/build-push-action@v4 - # with: - # context: ./packages - # file: ./packages/grid/backend/backend.dockerfile - # push: true - # platforms: linux/amd64,linux/arm64 - # tags: ${{ steps.meta_grid_backend.outputs.tags }} - # target: backend - # cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} - # cache-to: type=inline - - # - name: Docker meta for Grid-Frontend - # id: meta_grid_frontend - # uses: docker/metadata-action@v3 - # with: - # images: openmined/grid-frontend - # tags: | - # type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} - # type=raw,value=beta - - # - name: Build and push `grid-frontend` image to DockerHub - # uses: docker/build-push-action@v4 - # with: - # context: ./packages/grid/frontend - # file: ./packages/grid/frontend/frontend.dockerfile - # push: true - # platforms: linux/amd64,linux/arm64 - # tags: ${{ steps.meta_grid_frontend.outputs.tags }} - # target: grid-ui-development - # cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }} - # cache-to: type=inline - - # - name: Docker meta for Grid-Enclave - # id: meta_grid_enclave - # uses: docker/metadata-action@v3 - # with: - # images: openmined/grid-enclave - # tags: | - # type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} - # type=raw,value=beta - - # - name: Build and push `grid-enclave` image to DockerHub - # uses: docker/build-push-action@v4 - # with: - # context: ./packages - # file: ./packages/grid/worker/worker.dockerfile - # push: true - # platforms: linux/amd64,linux/arm64 - # tags: ${{ steps.meta_grid_enclave.outputs.tags }} - # target: worker - # cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} - # cache-to: type=inline + # Checkout Infra repo (nested) + - name: Checkout Infra Repo + uses: actions/checkout@v3 + with: + repository: ${{ secrets.INFRA_REPO }} + ref: "main" + token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }} + path: infrastructure + + # This step will copy the generated K8s manifest files to the correct directory in Infra repo + - name: Copy files to Infra Repo + run: | + if $is_pre_release; then + rm -rf infrastructure/gitops/environments/pre-release/. + cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/ + else + rm -rf infrastructure/gitops/environments/stable/. + cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/ + fi + + - name: Commit changes to Infra Repo + uses: EndBug/add-and-commit@v9 + with: + author_name: ${{ secrets.OM_BOT_NAME }} + author_email: ${{ secrets.OM_BOT_EMAIL }} + message: "Update K8s Manifests from Syft Repo" + add: "." + push: "origin main" + cwd: "./infrastructure/" + + - name: Create SyftCLI Config assets + run: | + pip install pyyaml + python scripts/create_syftcli_config.py + + - name: GitHub Release + uses: softprops/action-gh-release@v1 + with: + name: v${{ steps.release_checks.outputs.github_release_version }} + generate_release_notes: true + prerelease: ${{ steps.release_checks.outputs.is_pre_release }} + files: | + ./packages/syftcli/manifest.yml + ./build/syftcli-config/* + ./packages/hagrid/hagrid/manifest_template.yml + tag_name: v${{ steps.release_checks.outputs.github_release_version }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_LOGIN }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Docker meta for Grid-Backend + id: meta_grid_backend + uses: docker/metadata-action@v3 + with: + images: openmined/grid-backend + tags: | + type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} + type=raw,value=beta + + - name: Build and push `grid-backend` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages + file: ./packages/grid/backend/backend.dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta_grid_backend.outputs.tags }} + target: backend + cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} + cache-to: type=inline + + - name: Docker meta for Grid-Frontend + id: meta_grid_frontend + uses: docker/metadata-action@v3 + with: + images: openmined/grid-frontend + tags: | + type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} + type=raw,value=beta + + - name: Build and push `grid-frontend` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages/grid/frontend + file: ./packages/grid/frontend/frontend.dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta_grid_frontend.outputs.tags }} + target: grid-ui-development + cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }} + cache-to: type=inline + + - name: Docker meta for Grid-Enclave + id: meta_grid_enclave + uses: docker/metadata-action@v3 + with: + images: openmined/grid-enclave + tags: | + type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} + type=raw,value=beta + + - name: Build and push `grid-enclave` image to DockerHub + uses: docker/build-push-action@v4 + with: + context: ./packages + file: ./packages/grid/worker/worker.dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta_grid_enclave.outputs.tags }} + target: worker + cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} + cache-to: type=inline - name: Login to Docker-Helm run: | From 108480c5958558dcea542b2188bb5a8b61110b65 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:37:44 +0530 Subject: [PATCH 2/3] add cleanup of images --- .github/workflows/cd-syft.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index b5ffa680e89..a1673ebdb47 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -251,6 +251,15 @@ jobs: cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }} cache-to: type=inline + # free 10GB of space + - name: Remove unnecessary files + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker image prune --all --force + docker builder prune --all --force + docker system prune --all --force + - name: Docker meta for Grid-Enclave id: meta_grid_enclave uses: docker/metadata-action@v3 @@ -272,6 +281,15 @@ jobs: cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} cache-to: type=inline + # free 10GB of space + - name: Remove unnecessary files + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker image prune --all --force + docker builder prune --all --force + docker system prune --all --force + - name: Login to Docker-Helm run: | helm registry login registry-1.docker.io -u ${{ secrets.DOCKER_LOGIN }} --password ${{secrets.DOCKER_PASSWORD}} From 0fb53301a07ca5d434590e3e0a0240f2226626c1 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Wed, 1 Nov 2023 12:43:53 +0530 Subject: [PATCH 3/3] fixed dev parsing error for notes.txt --- packages/grid/helm/helm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index d3a7501f20a..9dd0ad1a618 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -143,7 +143,7 @@ def get_protocol_changes() -> str: protocol_changes = "" if protocol_path.exists(): - dev_protocol_changes = json.loads(protocol_path.read_text())["dev"] + dev_protocol_changes = json.loads(protocol_path.read_text()).get("dev", {}) protocol_changes = json.dumps( dev_protocol_changes.get("object_versions", {}), indent=4 )