[ci] fix syft.build.helm parse errors

.github/workflows/test-tox.yaml

@@ -0,0 +1,56 @@
+name: Test Tox
+
+on:
+  workflow_dispatch:
+    inputs:
+      toxenv:
+        type: choice
+        description: "Tox environment name to run"
+        required: true
+        options:
+          - syft.test.unit
+          - syft.test.notebook
+          - syft.test.security
+          - syft.build.helm
+          - syft.test.helm
+          - syftcli.test.unit
+          - syftcli.build
+          - frontend.test.unit
+
+jobs:
+  run-tox:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.11"
+
+      - name: Setup dependencies
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.0
+          DEVSPACE_VERSION=v6.3.3
+
+          # install tox
+          pip install --upgrade tox
+
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+
+          # Install devspace
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+
+      - name: Execute tox environment
+        shell: bash
+        run: |
+          tox -e ${{ github.event.inputs.toxenv }}

packages/grid/helm/helm.py

@@ -1,6 +1,7 @@
 # stdlib
 import argparse
 import os
+import shutil
 import sys
 from typing import Any
 
@@ -83,9 +84,9 @@ def fix_devspace_yaml(d: Any) -> None:
             fix_devspace_yaml(item)
 
 
-def get_yaml_name(doc: Any) -> Any:
+def get_yaml_name(doc: dict) -> Any:
     try:
-        return yaml.safe_load(doc).get("metadata", {}).get("name", "")
+        return doc.get("metadata", {}).get("name", "")
     except Exception:  # nosec
         return ""
 
@@ -97,13 +98,12 @@ def main() -> None:
         "file", nargs="?", type=argparse.FileType("r"), default=sys.stdin
     )
     args = parser.parse_args()
-    helm_dir = "helm"
     text = args.file.read()
-    file_count = 0
 
-    # input_file = f"{helm_dir}/raw_manifests.yaml"
-    # with open(input_file, "w") as f:
-    #     f.write(text)
+    file_count = 0
+    helm_dir = "helm"
+    manifest_file = f"{helm_dir}/manifests.yaml"
+    helm_chart_template_dir = f"{helm_dir}/syft/templates"
 
     # Read input from file or stdin
     lines = text.splitlines()
@@ -113,50 +113,43 @@ def main() -> None:
         first_index = next(
             i for i, line in enumerate(lines) if line.strip().startswith("apiVersion")
         )
-        input_data = "---\n" + "\n".join(lines[first_index - 1 :])
+        input_data = "\n".join(lines[first_index:])
     except StopIteration:
         print("❌ Error: No line starting with 'apiVersion' found in the input.")
-        print("------------------------------")
-        print("Got input text:")
-        print(text)
-        print("------------------------------")
         exit(1)
 
-    helm_chart_template_dir = f"{helm_dir}/syft/templates"
+    # Load the multi-doc yaml file
+    try:
+        yaml_docs = list(yaml.safe_load_all(input_data))
+    except Exception as e:
+        print(f"❌ Error while parsing yaml file: {e}")
+        exit(1)
+
+    # clear templates dir
+    shutil.rmtree(helm_chart_template_dir, ignore_errors=True)
 
-    # Split input_data into separate documents
-    yaml_docs = input_data.split("---")
+    # Create directories if they don't exist
+    os.makedirs(helm_chart_template_dir, exist_ok=True)
+
+    # Cleanup YAML docs
+    yaml_docs = [doc for doc in yaml_docs if doc]
 
     # Sort YAML docs based on metadata name
     yaml_docs.sort(key=get_yaml_name)
 
-    # Join sorted YAML docs
-    sorted_input_data = "---".join(yaml_docs)
-
     # Save sorted YAML docs to file
-    input_file = f"{helm_dir}/manifests.yaml"
-    with open(input_file, "w") as f:
-        f.write(sorted_input_data)
+    with open(manifest_file, "w") as f:
+        yaml.dump_all(yaml_docs, f)
+        # f.write(input_data)
 
     for doc in yaml_docs:
-        lines = doc.strip().split("\n")
-        if len(lines) <= 2:
-            continue  # skip empty sections
-
-        output_dir = os.path.join(helm_chart_template_dir)
-
-        # Create directories if they don't exist
-        os.makedirs(output_dir, exist_ok=True)
-
-        # Parse yaml to find metadata.name
-        yaml_content = yaml.safe_load("\n".join(lines))  # exclude source_line
-        fix_devspace_yaml(yaml_content)
-        name = yaml_content.get("metadata", {}).get("name")
-        kind = yaml_content.get("kind", "").lower()
+        fix_devspace_yaml(doc)
+        name = doc.get("metadata", {}).get("name")
+        kind = doc.get("kind", "").lower()
         if name:
             # Create new file with name or append if it already exists
-            new_file = os.path.join(output_dir, f"{name}-{kind}.yaml")
-            yaml_dump = yaml.dump(yaml_content)
+            new_file = os.path.join(helm_chart_template_dir, f"{name}-{kind}.yaml")
+            yaml_dump = yaml.dump(doc)
             yaml_dump = (
                 yaml_dump.replace("'{{", "{{")
                 .replace("}}'", "}}")
@@ -171,7 +164,7 @@ def main() -> None:
     if file_count > 0:
         print(f"✅ Done: Generated {file_count} template files")
     else:
-        print("❌ Failed: Generated zero files. Check input file for errors.")
+        print("❌ Failed: No files were generated. Check input for errors.")
         exit(1)
 
 

tox.ini

@@ -756,24 +756,20 @@ allowlist_externals =
 commands =
     k3d version
 
-    ; bash -c "docker rm $(docker ps -aq) --force || true"
     bash -c "k3d cluster delete build || true && \
         docker volume rm k3d-build-images --force || true"
 
     bash -c 'k3d cluster create build || true && \
         k3d cluster start build'
 
-    bash -c 'rm -rf packages/grid/helm/syft/templates/ && mkdir -p packages/grid/helm/syft/templates/'
-
-    bash -c 'rm -rf packages/grid/out.txt'
-
     bash -c 'cd packages/grid && \
         [[ -n "$CONTAINER_REGISTRY" ]] && REGISTRY_FLAG="--var CONTAINER_REGISTRY=$CONTAINER_REGISTRY" || REGISTRY_FLAG="" && \
         [[ -n "$VERSION" ]] && VERSION_FLAG="--var VERSION=$VERSION" || VERSION_FLAG="" && \
-        devspace deploy --render --skip-build --build-sequential --no-warn --silent ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" > out.txt'
+        devspace deploy --render --skip-build ${REGISTRY_FLAG} ${VERSION_FLAG} --kube-context "k3d-build" --no-warn --no-colors > out.txt ; \
+        printf '-%.0s' {1..100} ; echo "\nDEBUG out.txt:\n" ; cat out.txt ; printf '-%.0s' {1..100} ; echo'
 
     bash -c 'cd packages/grid && \
-        python3 helm/helm.py out.txt'
+        python3 helm/helm.py out.txt ; rm out.txt'
 
     bash -c 'cd packages/grid/helm && \
         helm lint syft'