diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 7493ffe23ad..9d13185d616 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -1,4 +1,4 @@ -name: CD - Syft - Deploy `dev` to K8s +name: CD - Kubernetes - Dev on: schedule: @@ -6,27 +6,44 @@ on: workflow_dispatch: inputs: - none: - description: "Run Tests Manually" - required: false + check-cache: + type: boolean + description: "Check workflow cache" + default: false + deploy-helm: + type: boolean + description: "Deploy Helm Charts" + default: false jobs: - deploy-syft-dev: + check-last-run: runs-on: om-ci-16vcpu-ubuntu2204 + + outputs: + same-commit: ${{ steps.cache.outputs.cache-hit }} + steps: - name: Check for new changes + if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true' id: cache - if: github.event_name == 'schedule' uses: actions/cache@v3 with: - path: scripts/commit_hash # we don't care about the file, just the key - key: dev-commit-${{ github.sha }} + path: scripts/k8s_dev_hash # we don't care about the file, just the key + key: k8s-dev-${{ github.sha }} lookup-only: true - - name: Quit if no new changes - if: github.event_name == 'schedule' && steps.cache.outputs.cache-hit == 'true' - run: exit 0 + - name: Save Commit SHA + if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true' + # only for making the cache action happy + run: echo "${{ github.sha }}" > scripts/k8s_dev_hash + + deploy-syft-dev: + needs: check-last-run + if: needs.check-last-run.outputs.same-commit != 'true' + + runs-on: om-ci-16vcpu-ubuntu2204 + steps: - name: Permission to home directory run: | sudo chown -R $USER:$USER $HOME @@ -51,10 +68,15 @@ jobs: id: buildx uses: docker/setup-buildx-action@v3 + - name: Install Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az version + - name: Login to Azure CLI uses: azure/login@v1 with: - creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}} + creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }} - name: Login to Azure Container Registry uses: azure/docker-login@v1 @@ -69,7 +91,7 @@ jobs: run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT - name: Build and push `grid-backend` image to registry - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./packages file: ./packages/grid/backend/backend.dockerfile @@ -78,9 +100,10 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-frontend` image to registry - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./packages/grid/frontend file: ./packages/grid/frontend/frontend.dockerfile @@ -88,11 +111,12 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} target: grid-ui-development # TODO: Re-enable once we have Enclave up and running # - name: Build and push `grid-enclave` image to registry - # uses: docker/build-push-action@v4 + # uses: docker/build-push-action@v5 # with: # context: ./packages # file: ./packages/grid/worker/worker.dockerfile @@ -101,9 +125,10 @@ jobs: # tags: | # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }} + # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-headscale` image to registry - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./packages/grid/vpn file: ./packages/grid/vpn/headscale.dockerfile @@ -111,9 +136,10 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-tailscale` image to registry - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./packages/grid/vpn file: ./packages/grid/vpn/tailscale.dockerfile @@ -121,9 +147,10 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }} - name: Build and push `grid-vpn-iptables` image to registry - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./packages/grid/vpn file: ./packages/grid/vpn/iptables.dockerfile @@ -131,8 +158,10 @@ jobs: tags: | ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }} + ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }} - name: Build Helm Chart & Copy to infra + if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true' shell: bash run: | K3D_VERSION=v5.6.0 @@ -159,6 +188,7 @@ jobs: cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/ - name: Commit & push changes to Infra Repo + if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true' uses: EndBug/add-and-commit@v9 with: author_name: ${{ secrets.OM_BOT_NAME }} @@ -169,22 +199,27 @@ jobs: cwd: "./infrastructure/" - name: Cleanup Azure Container Registry - uses: azure/CLI@v1 - with: - # SKIP_LINES = latest version dev & dev- (2 lines) + keep "n" previous version (n lines) + 1 - inlineScript: | - ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }} - REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv) + run: | + ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }} - KEEP_PREV_VERSIONS=1 - TAIL_FROM_LINE=$((2 + $KEEP_PREV_VERSIONS + 1)) + echo ">> Fetching repo list.." + REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv) - for repo in $REPO_LIST - do - echo "Cleaning up '$repo'" - az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv | tail -n +$TAIL_FROM_LINE | xargs -r -I% az acr repository delete --name $ACR_REGISTRY_NAME --image $repo:% --yes - done + KEEP_PREV_VERSIONS=5 + TAIL_FROM_LINE=$(($KEEP_PREV_VERSIONS + 1)) - - name: Save Commit SHA - # only for cache to hit - run: echo "${{ github.sha }}" > scripts/commit_hash + for repo in $REPO_LIST + do + echo "Cleaning up '$repo'" + az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv \ + | grep dev- \ + | tail -n +$TAIL_FROM_LINE \ + | xargs -r -I% az acr repository untag --name $ACR_REGISTRY_NAME --image $repo:% + done + + - name: Logout and cleanup Azure account + if: always() + run: | + az logout + az cache purge + az account clear diff --git a/packages/grid/helm/helm.py b/packages/grid/helm/helm.py index 1b8eed22949..fec1a2104c1 100644 --- a/packages/grid/helm/helm.py +++ b/packages/grid/helm/helm.py @@ -1,6 +1,7 @@ # stdlib import argparse import os +import shutil import sys from typing import Any @@ -83,9 +84,9 @@ def fix_devspace_yaml(d: Any) -> None: fix_devspace_yaml(item) -def get_yaml_name(doc: Any) -> Any: +def get_yaml_name(doc: dict) -> Any: try: - return yaml.safe_load(doc).get("metadata", {}).get("name", "") + return doc.get("metadata", {}).get("name", "") except Exception: # nosec return "" @@ -97,13 +98,12 @@ def main() -> None: "file", nargs="?", type=argparse.FileType("r"), default=sys.stdin ) args = parser.parse_args() - helm_dir = "helm" text = args.file.read() - file_count = 0 - # input_file = f"{helm_dir}/raw_manifests.yaml" - # with open(input_file, "w") as f: - # f.write(text) + file_count = 0 + helm_dir = "helm" + manifest_file = f"{helm_dir}/manifests.yaml" + helm_chart_template_dir = f"{helm_dir}/syft/templates" # Read input from file or stdin lines = text.splitlines() @@ -113,50 +113,42 @@ def main() -> None: first_index = next( i for i, line in enumerate(lines) if line.strip().startswith("apiVersion") ) - input_data = "---\n" + "\n".join(lines[first_index - 1 :]) + input_data = "\n".join(lines[first_index:]) except StopIteration: print("❌ Error: No line starting with 'apiVersion' found in the input.") - print("------------------------------") - print("Got input text:") - print(text) - print("------------------------------") exit(1) - helm_chart_template_dir = f"{helm_dir}/syft/templates" + # Load the multi-doc yaml file + try: + yaml_docs = list(yaml.safe_load_all(input_data)) + except Exception as e: + print(f"❌ Error while parsing yaml file: {e}") + exit(1) + + # clear templates dir + shutil.rmtree(helm_chart_template_dir, ignore_errors=True) - # Split input_data into separate documents - yaml_docs = input_data.split("---") + # Create directories if they don't exist + os.makedirs(helm_chart_template_dir, exist_ok=True) + + # Cleanup YAML docs + yaml_docs = [doc for doc in yaml_docs if doc] # Sort YAML docs based on metadata name yaml_docs.sort(key=get_yaml_name) - # Join sorted YAML docs - sorted_input_data = "---".join(yaml_docs) - # Save sorted YAML docs to file - input_file = f"{helm_dir}/manifests.yaml" - with open(input_file, "w") as f: - f.write(sorted_input_data) + with open(manifest_file, "w") as f: + yaml.dump_all(yaml_docs, f) for doc in yaml_docs: - lines = doc.strip().split("\n") - if len(lines) <= 2: - continue # skip empty sections - - output_dir = os.path.join(helm_chart_template_dir) - - # Create directories if they don't exist - os.makedirs(output_dir, exist_ok=True) - - # Parse yaml to find metadata.name - yaml_content = yaml.safe_load("\n".join(lines)) # exclude source_line - fix_devspace_yaml(yaml_content) - name = yaml_content.get("metadata", {}).get("name") - kind = yaml_content.get("kind", "").lower() + fix_devspace_yaml(doc) + name = doc.get("metadata", {}).get("name") + kind = doc.get("kind", "").lower() if name: # Create new file with name or append if it already exists - new_file = os.path.join(output_dir, f"{name}-{kind}.yaml") - yaml_dump = yaml.dump(yaml_content) + new_file = os.path.join(helm_chart_template_dir, f"{name}-{kind}.yaml") + yaml_dump = yaml.dump(doc) yaml_dump = ( yaml_dump.replace("'{{", "{{") .replace("}}'", "}}") @@ -171,7 +163,7 @@ def main() -> None: if file_count > 0: print(f"✅ Done: Generated {file_count} template files") else: - print("❌ Failed: Generated zero files. Check input file for errors.") + print("❌ Failed: No files were generated. Check input for errors.") exit(1) diff --git a/tox.ini b/tox.ini index 66531bccc70..357485e1b21 100644 --- a/tox.ini +++ b/tox.ini @@ -755,23 +755,25 @@ allowlist_externals = helm commands = k3d version + devspace version - ; bash -c "docker rm $(docker ps -aq) --force || true" - bash -c "k3d cluster delete build || true && \ + bash -c "k3d cluster delete build ; \ docker volume rm k3d-build-images --force || true" - bash -c 'k3d cluster create build || true && \ - k3d cluster start build' - - bash -c 'rm -rf packages/grid/helm/syft/templates/ && mkdir -p packages/grid/helm/syft/templates/' + bash -c 'k3d cluster create build && \ + k3d cluster start build && \ + echo "Waiting for cluster to be ready..." && \ + sleep 20' bash -c 'cd packages/grid && \ [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \ [[ -n "$VERSION" ]] && VERSION_FLAG="--var VERSION=$VERSION" || VERSION_FLAG="" && \ - devspace deploy --render --skip-build --no-warn --silent ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" > out.txt' + devspace deploy --render --skip-build ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" --no-warn --no-colors > out.txt ; \ + EXITCODE=$?; OUTPUT=$(cat out.txt); printf "Devspace exit code: $EXITCODE\nDevspace output:\n$OUTPUT\n"; exit $EXITCODE' bash -c 'cd packages/grid && \ - python3 helm/helm.py out.txt' + python3 helm/helm.py out.txt && \ + rm out.txt' bash -c 'cd packages/grid/helm && \ helm lint syft'