CH-95: remove alpine base, migration script that perform migration of…

… the applications based on a keyword and a dictionary of words to replace if the occurrence is found

docs/base-common-images.md

@@ -1,9 +1,10 @@
 # Define and use base and common images
 
 ## Relevant files and directory structure
- - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
- - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. 
-
+
+- `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
+- `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks.
+
 ## Base images and common images
 
 The main difference between the base images and common images is that base images are built in the root context, while
@@ -15,7 +16,7 @@ a specific purpose (e.g. enable widely used application stacks to inherit from).
 
 After generating the codeChange the Dockerfile in order to inherit from the main Docker image need to:
 
-1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located 
+1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located
 
 ```
 harness:
@@ -25,6 +26,7 @@ harness:
 ```
 
 2. Refer to the base image with the uppercased-underscored name of the dependency as an argument
+
 ```dockerfile
 ARG CLOUDHARNESS_BASE
 FROM $CLOUDHARNESS_BASE
@@ -39,24 +41,22 @@ In workflow tasks, the build dependency must be specified in the main applicatio
 ## Default images
 
 CloudHarness defines the following base images:
+
 - `cloudharness-base`: python-alpine with cloudharness common libraries preinstalled
-- `cloudharness-base-debian`: python-debian with cloudharness common libraries preinstalled
 - `cloudharness-django`: cloudharness-base with cloudharness django fastapi libraries preinstalled
-- `cloudharness-django-debian`: cloudharness-base-debian with cloudharness django fastapi libraries preinstalled
 - `cloudharness-fastapi`: cloudharness-base with fastapi libraries preinstalled
-- `cloudharness-fastapi-debian`: cloudharness-base-debian with fastapi libraries preinstalled
 
 Also the following common images are defined:
+
 - `cloudharness-flask`: common ground image to create Flask backends
 
 ## Override base and common images from CloudHarness
 
-To override a base or common image just create the same directory path in your 
+To override a base or common image just create the same directory path in your
 solution. The overriding can be used to replace files used in the build process or the Dockerfile itself.
 
 For example, overriding `cloudharness-base` could be useful to change some behaviour in the CloudHarness
 libraries or to provide new libraries to share within all applications.
 
 To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/base-images/cloudharness-base`
 then run `harness-deployment cloudharness MY_SOLUTION`
-

infrastructure/common-images/cloudharness-django/Dockerfile

@@ -1,5 +1,5 @@
-ARG CLOUDHARNESS_BASE_DEBIAN
-FROM $CLOUDHARNESS_BASE_DEBIAN
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
 
 ENV MODULE_NAME=backend
 ENV PORT=8080

infrastructure/common-images/cloudharness-django/README.md

@@ -1,3 +1,3 @@
-# CloudHarness-Django Base Debian image
+# CloudHarness-Django Base image
 
 Use this image to bring the package cloudharness-django into your image.

infrastructure/common-images/cloudharness-fastapi/Dockerfile

@@ -1,5 +1,5 @@
-ARG CLOUDHARNESS_BASE_DEBIAN
-FROM $CLOUDHARNESS_BASE_DEBIAN
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
 
 ENV MODULE_NAME=backend
 ENV PORT=8080

infrastructure/common-images/cloudharness-fastapi/README.md

@@ -1,3 +1,3 @@
-# CloudHarness-FastAPI Base Debian image
+# CloudHarness-FastAPI Base image
 
 Use this image for FastAPI based microservices.

libraries/cloudharness-utils/cloudharness_utils/constants.py

@@ -29,7 +29,7 @@
 VALUES_MANUAL_PATH = 'values.yaml'
 VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml'
 
-CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10-alpine', 'cloudharness-base-debian': 'python:3.9.10'}
+CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10'}
 
 
 CD_BUILD_STEP_BASE = 'build_base_images'
@@ -48,4 +48,4 @@
 E2E_TESTS_DIRNAME = 'e2e'
 API_TESTS_DIRNAME = 'api'
 
-E2E_TESTS_PROJECT_PATH = "test/test-e2e"
+E2E_TESTS_PROJECT_PATH = "test/test-e2e"

tools/deployment-cli-tools/ch_cli_tools/config/migration.json

@@ -0,0 +1,17 @@
+{
+    "deprecated": [
+        {
+            "keyword": "CLOUDHARNESS_BASE_DEBIAN",
+            "to_be_replaced": [
+                {
+                    "old": "CLOUDHARNESS_BASE_DEBIAN",
+                    "new": "CLOUDHARNESS_BASE"
+                },
+                {
+                    "old": "apk",
+                    "new": "apt"
+                }
+            ]
+        }
+    ]
+}

tools/deployment-cli-tools/ch_cli_tools/migration.py

@@ -0,0 +1,81 @@
+"""
+Utilities to perform a migration of the deployment to the latest supported version.
+"""
+import os
+import json
+
+from . import HERE
+from cloudharness_utils.constants import APPS_PATH
+from .utils import get_sub_paths, search_word_in_file, search_word_in_folder
+
+TO_CHECK = ['deploy', 'tasks', 'Dockerfile']
+
+def perform_migration(base_roots, accept_all=False):
+    all_files_detected = []
+    f = open(os.path.join(HERE, 'config', 'migration.json'), 'r')
+    migration_json = json.load(f)
+
+    for search_root in base_roots:
+        app_base_path = os.path.join(search_root, APPS_PATH)
+
+        # Iterate over all the applications to check if they need to be migrated
+        for app_path in get_sub_paths(app_base_path):
+            # Iterate the folders and files to check if they need to be migrated
+            for sub_path in TO_CHECK:
+                to_check = os.path.join(app_path, sub_path)
+                if os.path.isdir(to_check):
+                    for migration_obj in migration_json['deprecated']:
+                        files = search_word_in_folder(to_check, migration_obj['keyword'])
+                        for file in files:
+                            file_path = os.path.join(to_check, file)
+                            print('#########################################')
+                            print(f'Running migration on {file_path}')
+                            print('#########################################')
+                            all_files_detected.append(file_path)
+                            for word_to_replace in migration_obj['to_be_replaced']:
+                                read_file_and_replace(file_path, word_to_replace['old'], word_to_replace['new'], accept_all)
+                elif os.path.isfile(to_check):
+                    for migration_obj in migration_json['deprecated']:
+                        if len(search_word_in_file(to_check, migration_obj['keyword'])) > 0:
+                            print('#########################################')
+                            print(f'Running migration on {to_check}')
+                            print('#########################################')
+                            all_files_detected.append(to_check)
+                            for word_to_replace in migration_obj['to_be_replaced']:
+                                read_file_and_replace(to_check, word_to_replace['old'], word_to_replace['new'], accept_all)
+                else:
+                    pass
+                    # print(f'Path {to_check} does not exist')
+    print('=========================================')
+    print('=== Cloud Harness migration completed ===')
+    print('=========================================')
+    print('### Summary of the files to check post migration ###')
+    for file in all_files_detected:
+        print(f'{file}')
+
+
+def read_file_and_replace(file, old, new, accept_all=False):
+    file_p = open(file, 'r+')
+    lines = file_p.readlines()
+    file_p.seek(0)
+    for line in lines:
+        if old in line:
+            print(f'Found {old} in {file}')
+
+            if not accept_all:
+                print(f'Would you like to replace:')
+                print(line)
+                print(f'with:')
+                print(line.replace(old, new))
+                print('y/n')
+
+            if accept_all:
+                file_p.write(line.replace(old, new))
+            elif input() == 'y':
+                file_p.write(line.replace(old, new))
+            else:
+                file_p.write(line)
+        else:
+            file_p.write(line)
+    file_p.truncate()
+    file_p.close()

tools/deployment-cli-tools/ch_cli_tools/utils.py

@@ -376,3 +376,37 @@ def check_docker_manifest_exists(registry, image_name, tag, registry_secret=None
     api_url = f"https://{registry}/v2/{image_name}/manifests/{tag}"
     resp = requests.get(api_url)
     return resp.status_code == 200
+
+
+def filter_empty_strings(value):
+    return value != ""
+
+
+def search_word_in_file(file, word):
+    p = subprocess.Popen('grep -l %s %s'% (word, file), shell=True,
+        stdout=subprocess.PIPE)
+    output, _ = p.communicate()
+    output = output.decode("utf-8")
+    matches = []
+    if p.returncode == 1: # no matches found
+        pass
+    elif p.returncode == 0: # matches found
+        matches = output.split('\n')
+    else:
+        raise Exception(f'Migration Error: {file} grep failed with return code {p.returncode} and output {output}')
+    return list(filter(filter_empty_strings, matches))
+
+
+def search_word_in_folder(folder, word):
+    p = subprocess.Popen('grep -rl %s %s'% (word, '*'), shell=True,
+        stdout=subprocess.PIPE, cwd=folder)
+    output, _ = p.communicate()
+    output = output.decode("utf-8")
+    matches = []
+    if p.returncode == 1: # no matches found
+        pass
+    elif p.returncode == 0: # matches found
+        matches = output.split('\n')
+    else:
+        raise Exception(f'Migration Error: {folder} grep failed with return code {p.returncode} and output {output}')
+    return list(filter(filter_empty_strings, matches))

tools/deployment-cli-tools/harness-deployment

@@ -9,6 +9,7 @@ from ch_cli_tools.skaffold import create_skaffold_configuration, create_vscode_d
 from ch_cli_tools.codefresh import create_codefresh_deployment_scripts, write_env_file
 from ch_cli_tools.preprocessing import preprocess_build_overrides
 from ch_cli_tools.utils import merge_app_directories
+from ch_cli_tools.migration import perform_migration
 from cloudharness_utils.constants import DEPLOYMENT_PATH
 
 HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/')
@@ -61,7 +62,10 @@ if __name__ == "__main__":
                         help=f'Do not generate ci/cd files')
     parser.add_argument('-we', '--write-env', dest='write_env', action="store_const", default=None, const=True,
                             help=f'Write build env to .env file in {DEPLOYMENT_PATH}')
-
+    parser.add_argument('--migrate', dest='migration', action="store_true",
+                        help='Perform a migration of the deployment to the latest supported version.')
+    parser.add_argument('--migrate-accept-all', dest='migration_accept_all', action="store_true",
+                        help='Perform a migration of the deployment to the latest supported version without asking for confirmation. Use with caution.')
 
     args, unknown = parser.parse_known_args(sys.argv[1:])
 
@@ -74,13 +78,17 @@ if __name__ == "__main__":
         print('There are unknown args. Make sure to call the script with the accepted args. Try --help')
         print(f'unknown: {unknown}')
     else:
-
         if args.merge:
             logging.warn(
                 "Merge (-m, --merge) argument is deprecated. Directory merging is now set automatically")
             merge_app_directories(root_paths, destination=args.merge)
             root_paths = [args.merge]
 
+        if args.migration_accept_all:
+            perform_migration(root_paths, accept_all=True)
+        elif args.migration:
+            perform_migration(root_paths, accept_all=False)
+
         helm_values = create_helm_chart(
             root_paths,
             tag=args.tag,

tools/deployment-cli-tools/tests/resources/applications/migration_app/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_BASE_DEBIAN
+FROM $CLOUDHARNESS_BASE_DEBIAN
+
+RUN apk add --no-cache bash

tools/deployment-cli-tools/tests/resources/applications/migration_app/tasks/sub-task/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_BASE_DEBIAN
+FROM $CLOUDHARNESS_BASE_DEBIAN
+
+RUN apk add --no-cache bash

tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_BASE_DEBIAN
+FROM $CLOUDHARNESS_BASE_DEBIAN
+
+RUN apk add --no-cache bash

tools/deployment-cli-tools/tests/resources/migration/applications/migration_app/tasks/sub-task/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_BASE_DEBIAN
+FROM $CLOUDHARNESS_BASE_DEBIAN
+
+RUN apk add --no-cache bash

tools/deployment-cli-tools/tests/test_migration.py

@@ -0,0 +1,9 @@
+from ch_cli_tools.utils import *
+from ch_cli_tools.migration import perform_migration
+
+HERE = os.path.dirname(os.path.realpath(__file__)).replace(os.path.sep, '/')
+
+def test_migration_accept_all():
+    assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 2
+    perform_migration(os.path.join(HERE, './resources/migration'), accept_all=True)
+    assert len(search_word_in_folder(os.path.join(HERE, './resources/migration'), "CLOUDHARNESS_BASE_DEBIAN")) == 0

tools/deployment-cli-tools/tests/test_utils.py

@@ -64,4 +64,11 @@ def test_guess_build_dependencies_from_dockerfile():
 def test_check_docker_manifest_exists():
     assert check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "latest")
     assert not check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "RANDOM_TAG")
-
+
+
+def test_search_word_in_file():
+    assert len(search_word_in_file(os.path.join(HERE, './resources/applications/migration_app/Dockerfile'), "CLOUDHARNESS_BASE_DEBIAN")) == 1
+
+
+def test_search_word_in_folder():
+    assert len(search_word_in_folder(os.path.join(HERE, './resources/applications/migration_app/'), "CLOUDHARNESS_BASE_DEBIAN")) == 2