Feature - Move from alpine to debian image

.github/workflows/trivy-analysis.yml

@@ -33,15 +33,14 @@ jobs:
           DOCKER_BUILDKIT=1 docker build -t cloudharness-base:${{ github.sha }} . -f ./infrastructure/base-images/cloudharness-base/Dockerfile
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2
+        uses: aquasecurity/trivy-action@master
         with:
           image-ref: 'cloudharness-base:${{ github.sha }}'
-          format: 'template'
-          template: '@/contrib/sarif.tpl'
+          format: 'sarif'
           output: 'trivy-results.sarif'
           severity: 'CRITICAL,HIGH'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v1
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'

.gitignore

@@ -17,4 +17,6 @@ skaffold.yaml
 /deployment.yaml
 .hypothesis
 __pycache__
-.env
+.env
+tools/deployment-cli-tools/tests/resources/migration
+tools/deployment-cli-tools/tests/resources/migration.bak

applications/common/server/Dockerfile

@@ -6,8 +6,7 @@ ENV MODULE_NAME=common
 
 ENV WORKERS=2
 ENV PORT=8080
-RUN apk update
-RUN apk add postgresql-dev
+RUN apt update && apt install -y postgresql
 
 COPY ./requirements.txt /usr/src/app/
 
@@ -18,4 +17,4 @@ COPY . /usr/src/app
 ENV FLASK_ENV=production
 ENV APP_SETTINGS=common.config.ProductionConfig
 RUN pip3 install -e /usr/src/app
-ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app
+ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app

applications/workflows/tasks/extract-download/Dockerfile

@@ -1,6 +1,6 @@
-FROM alpine
-RUN apk update
-RUN apk add file unzip wget
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE as base
+RUN apt update && apt install -y file unzip wget
 
 ADD . /
 

docs/base-common-images.md

@@ -1,9 +1,10 @@
 # Define and use base and common images
 
 ## Relevant files and directory structure
- - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
- - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. 
-
+
+- `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
+- `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks.
+
 ## Base images and common images
 
 The main difference between the base images and common images is that base images are built in the root context, while
@@ -25,6 +26,7 @@ harness:
 ```
 
 2. Refer to the base image with the uppercased-underscored name of the dependency as an argument
+
 ```dockerfile
 ARG CLOUDHARNESS_BASE
 FROM $CLOUDHARNESS_BASE
@@ -39,24 +41,22 @@ In workflow tasks, the build dependency must be specified in the main applicatio
 ## Default images
 
 CloudHarness defines the following base images:
+
 - `cloudharness-base`: python-alpine with cloudharness common libraries preinstalled
-- `cloudharness-base-debian`: python-debian with cloudharness common libraries preinstalled
 - `cloudharness-django`: cloudharness-base with cloudharness django fastapi libraries preinstalled
-- `cloudharness-django-debian`: cloudharness-base-debian with cloudharness django fastapi libraries preinstalled
 - `cloudharness-fastapi`: cloudharness-base with fastapi libraries preinstalled
-- `cloudharness-fastapi-debian`: cloudharness-base-debian with fastapi libraries preinstalled
 
 Also the following common images are defined:
+
 - `cloudharness-flask`: common ground image to create Flask backends
 
 ## Override base and common images from CloudHarness
 
-To override a base or common image just create the same directory path in your 
+To override a base or common image just create the same directory path in your
 solution. The overriding can be used to replace files used in the build process or the Dockerfile itself.
 
 For example, overriding `cloudharness-base` could be useful to change some behaviour in the CloudHarness
 libraries or to provide new libraries to share within all applications.
 
 To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/base-images/cloudharness-base`
 then run `harness-deployment cloudharness MY_SOLUTION`
-

infrastructure/base-images/cloudharness-base/Dockerfile

@@ -1,17 +1,10 @@
-ARG PARENT=python:3.9.10-alpine
+ARG PARENT=python:3.9.10
 FROM ${PARENT}
 
-RUN apk update
-RUN apk upgrade
-
-# Add bash for k8s console
-RUN apk add bash
-# dev tools needed by some python libraries
-RUN apk add gcc libc-dev g++ python3-dev libffi-dev openssl-dev rust musl-dev cargo 
+RUN apt-get update && apt-get install -y nfs-common && rm -rf /var/lib/apt/lists/*
 
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
     pip install pytest  --prefer-binary
-
 COPY libraries/models/requirements.txt /libraries/models/requirements.txt
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
     pip install -r /libraries/models/requirements.txt  --prefer-binary
@@ -23,14 +16,13 @@ RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &
     pip install -r /libraries/client/cloudharness_cli/requirements.txt  --prefer-binary
 
 COPY libraries/models /libraries/models
-RUN pip install -e /libraries/models
+RUN pip install -e /libraries/models --no-cache-dir
 
 COPY libraries/cloudharness-common /libraries/cloudharness-common
 COPY libraries/client/cloudharness_cli /libraries/client/cloudharness_cli
 
-#
-
-RUN pip install -e /libraries/cloudharness-common
-RUN pip install -e /libraries/client/cloudharness_cli
+RUN pip install -e /libraries/models --no-cache-dir
+RUN pip install -e /libraries/cloudharness-common --no-cache-dir
+RUN pip install -e /libraries/client/cloudharness_cli --no-cache-dir
 
 WORKDIR /

infrastructure/common-images/cloudharness-django/Dockerfile

@@ -1,5 +1,5 @@
-ARG CLOUDHARNESS_BASE_DEBIAN
-FROM $CLOUDHARNESS_BASE_DEBIAN
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
 
 ENV MODULE_NAME=backend
 ENV PORT=8080

infrastructure/common-images/cloudharness-django/README.md

@@ -1,3 +1,3 @@
-# CloudHarness-Django Base Debian image
+# CloudHarness-Django Base image
 
 Use this image to bring the package cloudharness-django into your image.

infrastructure/common-images/cloudharness-fastapi/Dockerfile

@@ -1,5 +1,5 @@
-ARG CLOUDHARNESS_BASE_DEBIAN
-FROM $CLOUDHARNESS_BASE_DEBIAN
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
 
 ENV MODULE_NAME=backend
 ENV PORT=8080

infrastructure/common-images/cloudharness-fastapi/README.md

@@ -1,3 +1,3 @@
-# CloudHarness-FastAPI Base Debian image
+# CloudHarness-FastAPI Base image
 
 Use this image for FastAPI based microservices.

libraries/cloudharness-utils/cloudharness_utils/constants.py

@@ -29,7 +29,7 @@
 VALUES_MANUAL_PATH = 'values.yaml'
 VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml'
 
-CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10-alpine', 'cloudharness-base-debian': 'python:3.9.10'}
+CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10'}
 
 
 CD_BUILD_STEP_BASE = 'build_base_images'
@@ -48,4 +48,4 @@
 E2E_TESTS_DIRNAME = 'e2e'
 API_TESTS_DIRNAME = 'api'
 
-E2E_TESTS_PROJECT_PATH = "test/test-e2e"
+E2E_TESTS_PROJECT_PATH = "test/test-e2e"

tools/deployment-cli-tools/ch_cli_tools/config/migration.json

@@ -0,0 +1,46 @@
+{
+    "deprecated": [
+        {
+            "keyword": "CLOUDHARNESS_BASE_DEBIAN",
+            "to_be_replaced": [
+                {
+                    "old": "CLOUDHARNESS_BASE_DEBIAN",
+                    "new": "CLOUDHARNESS_BASE"
+                },
+                {
+                    "old": "apk add",
+                    "new": "apt-get install"
+                },
+                {
+                    "old": "apk update",
+                    "new": "apt-get update"
+                },
+                {
+                    "old": "apk upgrade",
+                    "new": "apt-get upgrade"
+                },
+                {
+                    "old": "apk add --no-cache",
+                    "new": "apt-get install --no-cache"
+                },
+                {
+                    "old": "apk update --no-cache",
+                    "new": "apt-get update --no-cache"
+                },
+                {
+                    "old": "apk upgrade --no-cache",
+                    "new": "apt-get upgrade --no-cache"
+                }
+            ]
+        },
+        {
+            "keyword": "cloudharness-base-debian",
+            "to_be_replaced": [
+                {
+                    "old": "cloudharness-base-debian",
+                    "new": "cloudharness-base"
+                }
+            ]
+        }
+    ]
+}

tools/deployment-cli-tools/ch_cli_tools/migration.py

@@ -0,0 +1,93 @@
+"""
+Utilities to perform a migration of the deployment to the latest supported version.
+"""
+import os
+import json
+
+from . import HERE
+from cloudharness_utils.constants import APPS_PATH
+from .utils import get_sub_paths, search_word_in_file, search_word_in_folder
+
+TO_CHECK = ["deploy", "tasks", "Dockerfile"]
+
+
+def perform_migration(base_root, accept_all=False):
+    all_files_detected = []
+    f = open(os.path.join(HERE, "config", "migration.json"), "r")
+    migration_json = json.load(f)
+
+    app_base_path = os.path.join(base_root, APPS_PATH)
+
+    # Iterate over all the applications to check if they need to be migrated
+    for app_path in get_sub_paths(app_base_path):
+        # Iterate the folders and files to check if they need to be migrated
+        for sub_path in TO_CHECK:
+            to_check = os.path.join(app_path, sub_path)
+            if os.path.isdir(to_check):
+                for migration_obj in migration_json["deprecated"]:
+                    files = search_word_in_folder(to_check, migration_obj["keyword"])
+                    for file in files:
+                        file_path = os.path.join(to_check, file)
+                        print("#########################################")
+                        print(f"Running migration on {file_path}")
+                        print("#########################################")
+                        all_files_detected.append(file_path)
+                        for word_to_replace in migration_obj["to_be_replaced"]:
+                            read_file_and_replace(
+                                file_path,
+                                word_to_replace["old"],
+                                word_to_replace["new"],
+                                accept_all,
+                            )
+            elif os.path.isfile(to_check):
+                for migration_obj in migration_json["deprecated"]:
+                    if len(search_word_in_file(to_check, migration_obj["keyword"])) > 0:
+                        print("#########################################")
+                        print(f"Running migration on {to_check}")
+                        print("#########################################")
+                        all_files_detected.append(to_check)
+                        for word_to_replace in migration_obj["to_be_replaced"]:
+                            read_file_and_replace(
+                                to_check,
+                                word_to_replace["old"],
+                                word_to_replace["new"],
+                                accept_all,
+                            )
+            else:
+                print(f'Path {to_check} does not exist')
+    print("=========================================")
+    print("=== Cloud Harness migration completed ===")
+    print("=========================================")
+    print("### Summary of the files to check post migration ###")
+    for file in all_files_detected:
+        print(f"{file}")
+    print("=========================================")
+    print("===           End of Summary          ===")
+    print("=========================================")
+
+
+def read_file_and_replace(file, old, new, accept_all=False):
+    file_p = open(file, "r+")
+    lines = file_p.readlines()
+    file_p.seek(0)
+    for line in lines:
+        if old in line:
+            print(f"Found {old} in {file}")
+
+            if not accept_all:
+                print(f"Would you like to replace:")
+                print(line)
+                print(f"with:")
+                print(line.replace(old, new))
+                print("y/n")
+
+            if accept_all:
+                file_p.write(line.replace(old, new))
+            elif input() == "y":
+                file_p.write(line.replace(old, new))
+            else:
+                file_p.write(line)
+        else:
+            file_p.write(line)
+    file_p.truncate()
+    file_p.close()