From ea8716679062d6c2f3ada202d157a98cda4eacf7 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:33:01 +0530 Subject: [PATCH] shift actions checkout position in cd pipeline --- .github/workflows/cd-feature-branch.yml | 540 ++++++++++++------------ 1 file changed, 270 insertions(+), 270 deletions(-) diff --git a/.github/workflows/cd-feature-branch.yml b/.github/workflows/cd-feature-branch.yml index 0b67328c5c7..56884fa05bc 100644 --- a/.github/workflows/cd-feature-branch.yml +++ b/.github/workflows/cd-feature-branch.yml @@ -47,10 +47,6 @@ jobs: server_version: ${{ steps.release_metadata.outputs.server_version }} steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.release_branch }} - # actions/setup-python doesn't yet support ARM - name: Setup Python on x64 if: ${{ !endsWith(matrix.runner, '-arm64') }} @@ -82,6 +78,10 @@ jobs: sudo apt-get update sudo apt-get install git -y + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.release_branch }} + - name: Check python version run: | python --version @@ -110,269 +110,269 @@ jobs: run: | python scripts/bump_version.py --bump-to-stable ${{ steps.release_metadata.outputs.server_version}} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_LOGIN }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push `syft-backend` image to DockerHub - id: syft-backend-build - uses: docker/build-push-action@v6 - with: - context: ./packages - file: ./packages/grid/backend/backend.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - target: backend - outputs: type=image,name=openmined/syft-backend,push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max - - - name: Export digest for syft-backend - run: | - mkdir -p /tmp/digests/syft-backend - digest="${{ steps.syft-backend-build.outputs.digest }}" - touch "/tmp/digests/syft-backend/${digest#sha256:}" - - - name: Build and push `syft-frontend` image to DockerHub - id: syft-frontend-build - uses: docker/build-push-action@v6 - with: - context: ./packages/grid/frontend - file: ./packages/grid/frontend/frontend.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - outputs: type=image,name=openmined/syft-frontend,push-by-digest=true,name-canonical=true,push=true - target: syft-ui-development - cache-from: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max - - - name: Export digest for syft-frontend - run: | - mkdir -p /tmp/digests/syft-frontend - digest="${{ steps.syft-frontend-build.outputs.digest }}" - touch "/tmp/digests/syft-frontend/${digest#sha256:}" - - - name: Build and push `syft-seaweedfs` image to DockerHub - id: syft-seaweedfs-build - uses: docker/build-push-action@v6 - with: - context: ./packages/grid/seaweedfs - file: ./packages/grid/seaweedfs/seaweedfs.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - outputs: type=image,name=openmined/syft-seaweedfs,push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max - - - name: Export digest for syft-seaweedfs - run: | - mkdir -p /tmp/digests/syft-seaweedfs - digest="${{ steps.syft-seaweedfs-build.outputs.digest }}" - touch "/tmp/digests/syft-seaweedfs/${digest#sha256:}" - - # Some of the dependencies of syft-enclave-attestation are not available for arm64 - # Hence, we are building syft-enclave-attestation only for x64 (see the `if` conditional) - - name: Build and push `syft-enclave-attestation` image to DockerHub - if: ${{ endsWith(matrix.runner, '-x64') }} - id: syft-enclave-attestation-build - uses: docker/build-push-action@v6 - with: - context: ./packages/grid/enclave/attestation - file: ./packages/grid/enclave/attestation/attestation.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - outputs: type=image,name=openmined/syft-enclave-attestation,push-by-digest=true,name-canonical=true,push=true - cache-from: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max - - - name: Export digest for syft-enclave-attestation - if: ${{ endsWith(matrix.runner, '-x64') }} - run: | - mkdir -p /tmp/digests/syft-enclave-attestation - digest="${{ steps.syft-enclave-attestation-build.outputs.digest }}" - touch "/tmp/digests/syft-enclave-attestation/${digest#sha256:}" - - - name: Build and push `syft` image to registry - id: syft-build - uses: docker/build-push-action@v6 - with: - context: ./packages/ - file: ./packages/grid/syft-client/syft.Dockerfile - outputs: type=image,name=openmined/syft-client,push-by-digest=true,name-canonical=true,push=true - platforms: ${{ steps.release_metadata.outputs.release_platform }} - cache-from: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }} - cache-to: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max - - - name: Export digest for `syft` image - run: | - mkdir -p /tmp/digests/syft - digest="${{ steps.syft-build.outputs.digest }}" - touch "/tmp/digests/syft/${digest#sha256:}" - - - name: Upload digests - uses: actions/upload-artifact@v4 - with: - name: digests-${{ steps.release_metadata.outputs.server_version }}-${{ steps.release_metadata.outputs.short_release_platform }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - #Used to merge x64 and arm64 into one docker image - merge-docker-images: - needs: [build-and-push-docker-images] - if: always() && (needs.build-and-push-docker-images.result == 'success') - - runs-on: sh-arc-linux-x64 - - outputs: - server_version: ${{ needs.build-and-push-docker-images.outputs.server_version }} - - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-${{ needs.build-and-push-docker-images.outputs.server_version }}-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_LOGIN }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Create manifest list and push for syft-backend - working-directory: /tmp/digests/syft-backend - run: | - docker buildx imagetools create \ - -t openmined/syft-backend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ - $(printf 'openmined/syft-backend@sha256:%s ' *) - - - name: Create manifest list and push for syft-frontend - working-directory: /tmp/digests/syft-frontend - run: | - docker buildx imagetools create \ - -t openmined/syft-frontend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ - $(printf 'openmined/syft-frontend@sha256:%s ' *) - - - name: Create manifest list and push for syft-seaweedfs - working-directory: /tmp/digests/syft-seaweedfs - run: | - docker buildx imagetools create \ - -t openmined/syft-seaweedfs:${{ needs.build-and-push-docker-images.outputs.server_version }} \ - $(printf 'openmined/syft-seaweedfs@sha256:%s ' *) - - - name: Create manifest list and push for syft-enclave-attestation - working-directory: /tmp/digests/syft-enclave-attestation - run: | - docker buildx imagetools create \ - -t openmined/syft-enclave-attestation:${{ needs.build-and-push-docker-images.outputs.server_version }} \ - $(printf 'openmined/syft-enclave-attestation@sha256:%s ' *) - - - name: Create manifest list and push for syft client - working-directory: /tmp/digests/syft - run: | - docker buildx imagetools create \ - -t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.server_version }} \ - $(printf 'openmined/syft-client@sha256:%s ' *) - - deploy-syft: - needs: [merge-docker-images] - if: always() && needs.merge-docker-images.result == 'success' - - runs-on: ubuntu-latest - - steps: - - name: Permission to home directory - run: | - sudo chown -R $USER:$USER $HOME - - - uses: actions/checkout@v4 - with: - token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} - ref: ${{ github.event.inputs.release_branch }} - - # free 10GB of space - - name: Remove unnecessary files - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker image prune --all --force - docker builder prune --all --force - docker system prune --all --force - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install uv==0.2.17 tox tox-uv==1.9.0 setuptools wheel twine bump2version PyYAML - uv --version - - - name: Bump to Final Release version - run: | - python scripts/bump_version.py --bump-to-stable ${{ needs.merge-docker-images.outputs.server_version }} - - - name: Build Helm Chart - shell: bash - run: | - # install k3d - K3D_VERSION=v5.6.3 - wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 - mv k3d-linux-amd64 k3d - chmod +x k3d - export PATH=`pwd`:$PATH - k3d version - - #Install Devspace - DEVSPACE_VERSION=v6.3.12 - curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace - chmod +x devspace - devspace version - - # Install helm - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm version - - tox -e syft.build.helm - tox -e syft.package.helm - - - name: Linting - run: | - tox -e lint || true - - - name: Manual Build and Publish - run: | - tox -e syft.publish - if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then - twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* - fi - - # Checkout to gh-pages and update helm repo - - name: Checkout to gh-pages - uses: actions/checkout@v4 - with: - ref: gh-pages - token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} - path: ghpages - - - name: Copy helm repo files from Syft Repo - run: | - rm -rf ghpages/helm/* - cp -R packages/grid/helm/repo/. ghpages/helm/ - - - name: Commit changes to gh-pages - uses: EndBug/add-and-commit@v9 - with: - author_name: ${{ secrets.OM_BOT_NAME }} - author_email: ${{ secrets.OM_BOT_EMAIL }} - message: "Update Helm package from Syft Repo" - add: "helm/" - push: "origin gh-pages" - cwd: "./ghpages/" + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Login to Docker + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKER_LOGIN }} + # password: ${{ secrets.DOCKER_PASSWORD }} + + # - name: Build and push `syft-backend` image to DockerHub + # id: syft-backend-build + # uses: docker/build-push-action@v6 + # with: + # context: ./packages + # file: ./packages/grid/backend/backend.dockerfile + # platforms: ${{ steps.release_metadata.outputs.release_platform }} + # target: backend + # outputs: type=image,name=openmined/syft-backend,push-by-digest=true,name-canonical=true,push=true + # cache-from: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }} + # cache-to: type=registry,ref=openmined/syft-backend:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max + + # - name: Export digest for syft-backend + # run: | + # mkdir -p /tmp/digests/syft-backend + # digest="${{ steps.syft-backend-build.outputs.digest }}" + # touch "/tmp/digests/syft-backend/${digest#sha256:}" + + # - name: Build and push `syft-frontend` image to DockerHub + # id: syft-frontend-build + # uses: docker/build-push-action@v6 + # with: + # context: ./packages/grid/frontend + # file: ./packages/grid/frontend/frontend.dockerfile + # platforms: ${{ steps.release_metadata.outputs.release_platform }} + # outputs: type=image,name=openmined/syft-frontend,push-by-digest=true,name-canonical=true,push=true + # target: syft-ui-development + # cache-from: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform }} + # cache-to: type=registry,ref=openmined/syft-frontend:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max + + # - name: Export digest for syft-frontend + # run: | + # mkdir -p /tmp/digests/syft-frontend + # digest="${{ steps.syft-frontend-build.outputs.digest }}" + # touch "/tmp/digests/syft-frontend/${digest#sha256:}" + + # - name: Build and push `syft-seaweedfs` image to DockerHub + # id: syft-seaweedfs-build + # uses: docker/build-push-action@v6 + # with: + # context: ./packages/grid/seaweedfs + # file: ./packages/grid/seaweedfs/seaweedfs.dockerfile + # platforms: ${{ steps.release_metadata.outputs.release_platform }} + # outputs: type=image,name=openmined/syft-seaweedfs,push-by-digest=true,name-canonical=true,push=true + # cache-from: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform }} + # cache-to: type=registry,ref=openmined/syft-seaweedfs:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max + + # - name: Export digest for syft-seaweedfs + # run: | + # mkdir -p /tmp/digests/syft-seaweedfs + # digest="${{ steps.syft-seaweedfs-build.outputs.digest }}" + # touch "/tmp/digests/syft-seaweedfs/${digest#sha256:}" + + # # Some of the dependencies of syft-enclave-attestation are not available for arm64 + # # Hence, we are building syft-enclave-attestation only for x64 (see the `if` conditional) + # - name: Build and push `syft-enclave-attestation` image to DockerHub + # if: ${{ endsWith(matrix.runner, '-x64') }} + # id: syft-enclave-attestation-build + # uses: docker/build-push-action@v6 + # with: + # context: ./packages/grid/enclave/attestation + # file: ./packages/grid/enclave/attestation/attestation.dockerfile + # platforms: ${{ steps.release_metadata.outputs.release_platform }} + # outputs: type=image,name=openmined/syft-enclave-attestation,push-by-digest=true,name-canonical=true,push=true + # cache-from: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform }} + # cache-to: type=registry,ref=openmined/syft-enclave-attestation:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max + + # - name: Export digest for syft-enclave-attestation + # if: ${{ endsWith(matrix.runner, '-x64') }} + # run: | + # mkdir -p /tmp/digests/syft-enclave-attestation + # digest="${{ steps.syft-enclave-attestation-build.outputs.digest }}" + # touch "/tmp/digests/syft-enclave-attestation/${digest#sha256:}" + + # - name: Build and push `syft` image to registry + # id: syft-build + # uses: docker/build-push-action@v6 + # with: + # context: ./packages/ + # file: ./packages/grid/syft-client/syft.Dockerfile + # outputs: type=image,name=openmined/syft-client,push-by-digest=true,name-canonical=true,push=true + # platforms: ${{ steps.release_metadata.outputs.release_platform }} + # cache-from: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }} + # cache-to: type=registry,ref=openmined/syft-client:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max + + # - name: Export digest for `syft` image + # run: | + # mkdir -p /tmp/digests/syft + # digest="${{ steps.syft-build.outputs.digest }}" + # touch "/tmp/digests/syft/${digest#sha256:}" + + # - name: Upload digests + # uses: actions/upload-artifact@v4 + # with: + # name: digests-${{ steps.release_metadata.outputs.server_version }}-${{ steps.release_metadata.outputs.short_release_platform }} + # path: /tmp/digests/* + # if-no-files-found: error + # retention-days: 1 + + # #Used to merge x64 and arm64 into one docker image + # merge-docker-images: + # needs: [build-and-push-docker-images] + # if: always() && (needs.build-and-push-docker-images.result == 'success') + + # runs-on: sh-arc-linux-x64 + + # outputs: + # server_version: ${{ needs.build-and-push-docker-images.outputs.server_version }} + + # steps: + # - name: Download digests + # uses: actions/download-artifact@v4 + # with: + # path: /tmp/digests + # pattern: digests-${{ needs.build-and-push-docker-images.outputs.server_version }}-* + # merge-multiple: true + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Login to Docker + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKER_LOGIN }} + # password: ${{ secrets.DOCKER_PASSWORD }} + + # - name: Create manifest list and push for syft-backend + # working-directory: /tmp/digests/syft-backend + # run: | + # docker buildx imagetools create \ + # -t openmined/syft-backend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ + # $(printf 'openmined/syft-backend@sha256:%s ' *) + + # - name: Create manifest list and push for syft-frontend + # working-directory: /tmp/digests/syft-frontend + # run: | + # docker buildx imagetools create \ + # -t openmined/syft-frontend:${{ needs.build-and-push-docker-images.outputs.server_version }} \ + # $(printf 'openmined/syft-frontend@sha256:%s ' *) + + # - name: Create manifest list and push for syft-seaweedfs + # working-directory: /tmp/digests/syft-seaweedfs + # run: | + # docker buildx imagetools create \ + # -t openmined/syft-seaweedfs:${{ needs.build-and-push-docker-images.outputs.server_version }} \ + # $(printf 'openmined/syft-seaweedfs@sha256:%s ' *) + + # - name: Create manifest list and push for syft-enclave-attestation + # working-directory: /tmp/digests/syft-enclave-attestation + # run: | + # docker buildx imagetools create \ + # -t openmined/syft-enclave-attestation:${{ needs.build-and-push-docker-images.outputs.server_version }} \ + # $(printf 'openmined/syft-enclave-attestation@sha256:%s ' *) + + # - name: Create manifest list and push for syft client + # working-directory: /tmp/digests/syft + # run: | + # docker buildx imagetools create \ + # -t openmined/syft-client:${{ needs.build-and-push-docker-images.outputs.server_version }} \ + # $(printf 'openmined/syft-client@sha256:%s ' *) + + # deploy-syft: + # needs: [merge-docker-images] + # if: always() && needs.merge-docker-images.result == 'success' + + # runs-on: ubuntu-latest + + # steps: + # - name: Permission to home directory + # run: | + # sudo chown -R $USER:$USER $HOME + + # - uses: actions/checkout@v4 + # with: + # token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} + # ref: ${{ github.event.inputs.release_branch }} + + # # free 10GB of space + # - name: Remove unnecessary files + # run: | + # sudo rm -rf /usr/share/dotnet + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # docker image prune --all --force + # docker builder prune --all --force + # docker system prune --all --force + + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: "3.12" + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install uv==0.2.17 tox tox-uv==1.9.0 setuptools wheel twine bump2version PyYAML + # uv --version + + # - name: Bump to Final Release version + # run: | + # python scripts/bump_version.py --bump-to-stable ${{ needs.merge-docker-images.outputs.server_version }} + + # - name: Build Helm Chart + # shell: bash + # run: | + # # install k3d + # K3D_VERSION=v5.6.3 + # wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 + # mv k3d-linux-amd64 k3d + # chmod +x k3d + # export PATH=`pwd`:$PATH + # k3d version + + # #Install Devspace + # DEVSPACE_VERSION=v6.3.12 + # curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace + # chmod +x devspace + # devspace version + + # # Install helm + # curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # helm version + + # tox -e syft.build.helm + # tox -e syft.package.helm + + # - name: Linting + # run: | + # tox -e lint || true + + # - name: Manual Build and Publish + # run: | + # tox -e syft.publish + # if [[ "${{ github.event.inputs.release_platform }}" == "TEST_PYPI" ]]; then + # twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/* + # fi + + # # Checkout to gh-pages and update helm repo + # - name: Checkout to gh-pages + # uses: actions/checkout@v4 + # with: + # ref: gh-pages + # token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }} + # path: ghpages + + # - name: Copy helm repo files from Syft Repo + # run: | + # rm -rf ghpages/helm/* + # cp -R packages/grid/helm/repo/. ghpages/helm/ + + # - name: Commit changes to gh-pages + # uses: EndBug/add-and-commit@v9 + # with: + # author_name: ${{ secrets.OM_BOT_NAME }} + # author_email: ${{ secrets.OM_BOT_EMAIL }} + # message: "Update Helm package from Syft Repo" + # add: "helm/" + # push: "origin gh-pages" + # cwd: "./ghpages/"