Merge pull request #3 from MetaCell/develop

Develop

.gitignore

@@ -7,3 +7,5 @@ node_modules
 .coverage
 *.DS_Store
 deployment
+*.egg-info
+*.idea

blueprint/deployment-configuration/codefresh-template.yaml

@@ -0,0 +1,76 @@
+version: '1.0'
+stages:
+- prepare
+- build
+- deploy
+steps:
+  main_clone:
+    title: Clone main repository
+    type: git-clone
+    stage: prepare
+    repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}'
+    revision: '${{CF_BRANCH}}'
+    git: github
+  post_main_clone:
+    title: Post main clone
+    type: parallel
+    stage: build
+    steps:
+      - title: Cloning cloud-harness repository...
+        type: git-clone
+        stage: prepare
+        repo: 'https://github.com/MetaCell/cloud-harness.git'
+        revision: '${{CLOUDHARNESS_BRANCH}}'
+        git: github
+  build_base_images:
+    title: Build base images
+    type: parallel
+    stage: build
+    steps:
+      REPLACE_ME
+  build_static_images:
+    title: Build static images
+    type: parallel
+    stage: build
+    steps:
+      REPLACE_ME
+  build_application_images:
+    type: parallel
+    stage: build
+    steps:
+      REPLACE_ME
+
+  prepare_deployment:
+    title: "Prepare helm chart"
+    image: python:3.7
+    stage: deploy
+    working_directory: .
+    commands:
+      - pip install -r cloud-harness/requirements.txt
+      - harness-deployment . cloud-harness -t ${{CF_REVISION}} -d ${{DOMAIN}}
+
+  prepare_deployment_view:
+    commands:
+      - 'helm template ./deployment/helm --debug -n ${{NAME}}'
+    environment:
+      - ACTION=auth
+      - KUBE_CONTEXT=${{NAME}}
+    image: codefresh/cfstep-helm:2.16.1
+    stage: prepare
+    title: 'View helm chart'
+
+  deployment:
+    stage: deploy
+    image: 'codefresh/cfstep-helm:2.16.1'
+    title: Installing chart
+    environment:
+      - CHART_REF=./deployment/helm
+      - RELEASE_NAME=${{NAME}}
+      - KUBE_CONTEXT=${{NAME}}
+      - NAMESPACE=${{NAMESPACE}}
+      - TILLER_NAMESPACE=kube-system
+      - CHART_VERSION=0.0.1
+      - HELM_REPO_USE_HTTP=false
+      - HELM_REPO_CONTEXT_PATH=
+      - TIMEOUT=600
+      - VALUESFILE_values=./deployment/helm/values.yaml

blueprint/deployment-configuration/helm/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: 0.0.1
+description: 'CHANGE DESCRIPTION'
+name: 'CHANGE ME'
+version: 0.0.1

blueprint/deployment-configuration/values-template.yaml

@@ -0,0 +1,2 @@
+domain: CHANGE ME
+namespace: CHANGE ME

blueprint/infrastructure/README.md

@@ -0,0 +1,18 @@
+# Infrastructure
+
+Here we put all the resources intended to install and deploy the platform on Kubernetes.
+
+## Relevant files and directory structure
+ - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
+ - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. 
+
+## Base images and common images
+
+The main difference between the base images and common images is that base images are built in the root context, while
+common images are built in a local context.
+So, base images are general purpose and are mainly used to provide access to custom libraries, while common images can have
+a specific purpose (e.g. enable widely used libraries for tasks).
+
+
+
+

blueprint/infrastructure/base-images/README.md

@@ -0,0 +1,3 @@
+# Base images
+
+Here we find base images to inherit from our applications

blueprint/infrastructure/common-images/README.md

@@ -0,0 +1,3 @@
+#Common images
+
+Here we find common images reused in tasks and applications

requirements.txt

@@ -1,2 +1,2 @@
--e utilities/cloudharness-deploy
-pyaml
+-e utilities
+oyaml

utilities/cloudharness_utilities/codefresh.py

@@ -1,48 +1,65 @@
 import os
-import pyaml
-import yaml
+import oyaml as yaml
 import logging
 
 from .constants import HERE, BUILD_STEP_BASE, BUILD_STEP_STATIC, BUILD_STEP_PARALLEL, BUILD_STEP_INSTALL, \
     CODEFRESH_REGISTRY, K8S_IMAGE_EXCLUDE, CODEFRESH_PATH, CODEFRESH_BUILD_PATH, \
-    CODEFRESH_TEMPLATE_PATH, APPS_PATH, STATIC_IMAGES_PATH, BASE_IMAGES_PATH
+    CODEFRESH_TEMPLATE_PATH, APPS_PATH, STATIC_IMAGES_PATH, BASE_IMAGES_PATH, DEPLOYMENT_PATH
 from .helm import collect_helm_values
 from .utils import find_dockerfiles_paths, image_name_from_docker_path, \
     get_image_name, get_template, merge_to_yaml_file
 
 logging.getLogger().setLevel(logging.INFO)
 
-def create_codefresh_deployment_scripts(deployment_root, tag="${{CF_REVISION}}", codefresh_path=CODEFRESH_PATH):
+CLOUD_HARNESS_PATH = 'cloud-harness'
+
+
+def create_codefresh_deployment_scripts(deployment_root_path, tag="${{CF_REVISION}}", codefresh_path=CODEFRESH_PATH):
     """
     Entry point to create deployment scripts for codefresh: codefresh.yaml and helm chart
     """
 
-    codefresh = get_template(CODEFRESH_TEMPLATE_PATH)
+    codefresh = get_template(os.path.join(deployment_root_path, CODEFRESH_TEMPLATE_PATH))
 
     codefresh['steps'][BUILD_STEP_BASE]['steps'] = {}
     codefresh['steps'][BUILD_STEP_STATIC]['steps'] = {}
     codefresh['steps'][BUILD_STEP_PARALLEL]['steps'] = {}
 
     def codefresh_build_step_from_base_path(base_path, build_step, root_context=None):
-        abs_base_path = os.path.join(deployment_root, base_path)
+        abs_base_path = os.path.join(deployment_root_path, base_path)
         for dockerfile_path in find_dockerfiles_paths(abs_base_path):
-            app_relative_to_root = os.path.relpath(dockerfile_path, deployment_root)
+            app_relative_to_root = os.path.relpath(dockerfile_path, deployment_root_path)
             app_relative_to_base = os.path.relpath(dockerfile_path, abs_base_path)
             app_name = image_name_from_docker_path(app_relative_to_base)
             if app_name in K8S_IMAGE_EXCLUDE:
                 continue
-            build = codefresh_app_build_spec(app_name, os.path.relpath(root_context, deployment_root) if root_context else app_relative_to_root,
-                                             dockerfile_path=os.path.join(os.path.relpath(dockerfile_path, root_context) if root_context else '', "Dockerfile"))
+            build = codefresh_app_build_spec(app_name, os.path.relpath(root_context,
+                                                                       deployment_root_path) if root_context else app_relative_to_root,
+                                             dockerfile_path=os.path.join(
+                                                 os.path.relpath(dockerfile_path, root_context) if root_context else '',
+                                                 "Dockerfile"))
             codefresh['steps'][build_step]['steps'][app_name] = build
 
-    codefresh_build_step_from_base_path(BASE_IMAGES_PATH, BUILD_STEP_BASE, root_context=deployment_root)
+    codefresh_build_step_from_base_path(BASE_IMAGES_PATH, BUILD_STEP_BASE, root_context=deployment_root_path)
     codefresh_build_step_from_base_path(STATIC_IMAGES_PATH, BUILD_STEP_STATIC)
     codefresh_build_step_from_base_path(APPS_PATH, BUILD_STEP_PARALLEL)
 
-    codefresh['steps'] = {k:step for k, step in codefresh['steps'].items() if 'type' not in step or step['type'] != 'parallel' or step['steps']}
+    if os.path.exists(os.path.join(deployment_root_path, CLOUD_HARNESS_PATH)):
+        logging.info('Create build steps for cloud-harness images')
+        codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, BASE_IMAGES_PATH), BUILD_STEP_BASE,
+                                            root_context=deployment_root_path)
+        codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, STATIC_IMAGES_PATH), BUILD_STEP_STATIC)
+        codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, APPS_PATH), BUILD_STEP_PARALLEL)
+
+    codefresh['steps'] = {k: step for k, step in codefresh['steps'].items() if
+                          'type' not in step or step['type'] != 'parallel' or (step['steps'] if 'steps' in step else [])}
 
-    with open(codefresh_path, 'w') as f:
-        pyaml.dump(codefresh, f)
+    codefresh_abs_path = os.path.join(deployment_root_path, DEPLOYMENT_PATH, codefresh_path)
+    codefresh_dir = os.path.dirname(codefresh_abs_path)
+    if not os.path.exists(codefresh_dir):
+        os.makedirs(codefresh_dir)
+    with open(codefresh_abs_path, 'w') as f:
+        yaml.dump(codefresh, f)
 
 
 def codefresh_build_spec(**kwargs):
@@ -60,7 +77,8 @@ def codefresh_build_spec(**kwargs):
 def codefresh_app_build_spec(app_name, app_path, dockerfile_path="Dockerfile"):
     logging.info('Generating build script for ' + app_name)
     title = app_name.capitalize().replace('-', ' ').replace('/', ' ').replace('.', ' ').strip()
-    build = codefresh_build_spec(image_name=get_image_name(app_name), title=title, working_directory= './' + app_path, dockerfile=dockerfile_path)
+    build = codefresh_build_spec(image_name=get_image_name(app_name), title=title, working_directory='./' + app_path,
+                                 dockerfile=dockerfile_path)
 
     specific_build_template_path = os.path.join(app_path, 'build.yaml')
     if os.path.exists(specific_build_template_path):

utilities/cloudharness_utilities/constants.py

@@ -13,23 +13,24 @@
 BASE_IMAGES_PATH = os.path.join(INFRASTRUCTURE_PATH, 'base-images')
 NEUTRAL_PATHS = ('src', 'tasks', 'server')
 APPS_PATH = 'applications'
-
+DEPLOYMENT_PATH = 'deployment'
 CODEFRESH_PATH = 'codefresh/codefresh.yaml'
 
-CODEFRESH_BUILD_PATH = 'deployment-templates/codefresh-build-template.yaml'
-CODEFRESH_TEMPLATE_PATH = 'deployment-templates/codefresh-template.yaml'
+DEPLOYMENT_CONFIGURATION_PATH = 'deployment-configuration'
+
+CODEFRESH_BUILD_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/codefresh-build-template.yaml'
+CODEFRESH_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/codefresh-template.yaml'
 CODEFRESH_REGISTRY = "r.cfcr.io/tarelli"
 
 VALUES_MANUAL_PATH = 'values.yaml'
-VALUE_TEMPLATE_PATH = 'deployment-templates/value-template.yaml'
+VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml'
 
 CH_BASE_IMAGES = {'cloudharness-base': 'python:3.7-alpine', 'cloudharness-base-debian': 'python:3'}
 
 K8S_IMAGE_EXCLUDE = ('accounts-keycloak-gatekeeper',)
 
-BUILD_STEP_BASE = 'x1_build_base_image'
-BUILD_STEP_STATIC = 'x2_static_build'
-BUILD_STEP_PARALLEL = 'x3_parallel_build'
-BUILD_STEP_INSTALL = 'x4_deployment'
+BUILD_STEP_BASE = 'build_base_images'
+BUILD_STEP_STATIC = 'build_static_images'
+BUILD_STEP_PARALLEL = 'build_application_images'
+BUILD_STEP_INSTALL = 'deployment'
 
-DEPLOYMENT_CONFIGURATION_PATH = 'deployment-configuration'

utilities/cloudharness_utilities/deployment-configuration/codefresh-template.yaml

@@ -5,47 +5,54 @@ stages:
 - deploy
 steps:
   main_clone:
-    title: Cloning main repository...
+    title: Clone main repository
     type: git-clone
     stage: prepare
     repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}'
     revision: '${{CF_BRANCH}}'
     git: github
-  x1_build_base_image:
+  post_main_clone:
+    title: Post main clone
+    type: parallel
+    stage: prepare
+  build_base_images:
+    title: Build base images
     type: parallel
     stage: build
     steps:
       REPLACE_ME
-  x2_static_build:
+  build_static_images:
+    title: Build static images
     type: parallel
     stage: build
     steps:
       REPLACE_ME
-  x3_parallel_build:
+  build_application_images:
     type: parallel
     stage: build
     steps:
       REPLACE_ME
 
   prepare_deployment:
     title: "Prepare helm chart"
-    image: r.cfcr.io/tarelli/cloudharness-deploy
+    image: python:3.7
     stage: deploy
-    working_directory: ./deployment
+    working_directory: .
     commands:
-      - python deployment-generate.py
+      - pip install -r cloud-harness/requirements.txt
+      - harness-deployment . cloud-harness -t ${{CF_REVISION}} -d ${{DOMAIN}}
 
-  prepare_deployment_view2:
+  prepare_deployment_view:
     commands:
-      - 'helm template ./deployment/helm --debug -n '${{NAME}}
+      - 'helm template ./deployment/helm --debug -n ${{NAME}}'
     environment:
       - ACTION=auth
       - KUBE_CONTEXT=${{NAME}}
     image: codefresh/cfstep-helm:2.16.1
     stage: prepare
     title: 'View helm chart'
 
-  x4_deployment:
+  deployment:
     stage: deploy
     image: 'codefresh/cfstep-helm:2.16.1'
     title: Installing chart
@@ -59,9 +66,4 @@ steps:
       - HELM_REPO_USE_HTTP=false
       - HELM_REPO_CONTEXT_PATH=
       - TIMEOUT=600
-      - VALUESFILE_values=./deployment/helm/values.yaml
-
-    when:
-      branch:
-        only: 
-          - '${{CF_BRANCH}}'
+      - VALUESFILE_values=./deployment/helm/values.yaml