diff --git a/.gitignore b/.gitignore index 76933e9d..a39b23fe 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ node_modules .coverage *.DS_Store deployment +*.egg-info +*.idea diff --git a/blueprint/deployment-configuration/codefresh-template.yaml b/blueprint/deployment-configuration/codefresh-template.yaml new file mode 100644 index 00000000..c2cc568a --- /dev/null +++ b/blueprint/deployment-configuration/codefresh-template.yaml @@ -0,0 +1,76 @@ +version: '1.0' +stages: +- prepare +- build +- deploy +steps: + main_clone: + title: Clone main repository + type: git-clone + stage: prepare + repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' + revision: '${{CF_BRANCH}}' + git: github + post_main_clone: + title: Post main clone + type: parallel + stage: build + steps: + - title: Cloning cloud-harness repository... + type: git-clone + stage: prepare + repo: 'https://github.com/MetaCell/cloud-harness.git' + revision: '${{CLOUDHARNESS_BRANCH}}' + git: github + build_base_images: + title: Build base images + type: parallel + stage: build + steps: + REPLACE_ME + build_static_images: + title: Build static images + type: parallel + stage: build + steps: + REPLACE_ME + build_application_images: + type: parallel + stage: build + steps: + REPLACE_ME + + prepare_deployment: + title: "Prepare helm chart" + image: python:3.7 + stage: deploy + working_directory: . + commands: + - pip install -r cloud-harness/requirements.txt + - harness-deployment . cloud-harness -t ${{CF_REVISION}} -d ${{DOMAIN}} + + prepare_deployment_view: + commands: + - 'helm template ./deployment/helm --debug -n ${{NAME}}' + environment: + - ACTION=auth + - KUBE_CONTEXT=${{NAME}} + image: codefresh/cfstep-helm:2.16.1 + stage: prepare + title: 'View helm chart' + + deployment: + stage: deploy + image: 'codefresh/cfstep-helm:2.16.1' + title: Installing chart + environment: + - CHART_REF=./deployment/helm + - RELEASE_NAME=${{NAME}} + - KUBE_CONTEXT=${{NAME}} + - NAMESPACE=${{NAMESPACE}} + - TILLER_NAMESPACE=kube-system + - CHART_VERSION=0.0.1 + - HELM_REPO_USE_HTTP=false + - HELM_REPO_CONTEXT_PATH= + - TIMEOUT=600 + - VALUESFILE_values=./deployment/helm/values.yaml \ No newline at end of file diff --git a/blueprint/deployment-configuration/helm/Chart.yaml b/blueprint/deployment-configuration/helm/Chart.yaml new file mode 100644 index 00000000..f248ab72 --- /dev/null +++ b/blueprint/deployment-configuration/helm/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: 0.0.1 +description: 'CHANGE DESCRIPTION' +name: 'CHANGE ME' +version: 0.0.1 diff --git a/blueprint/deployment-configuration/values-template.yaml b/blueprint/deployment-configuration/values-template.yaml new file mode 100644 index 00000000..be27d78d --- /dev/null +++ b/blueprint/deployment-configuration/values-template.yaml @@ -0,0 +1,2 @@ +domain: CHANGE ME +namespace: CHANGE ME diff --git a/blueprint/infrastructure/README.md b/blueprint/infrastructure/README.md new file mode 100644 index 00000000..940966a9 --- /dev/null +++ b/blueprint/infrastructure/README.md @@ -0,0 +1,18 @@ +# Infrastructure + +Here we put all the resources intended to install and deploy the platform on Kubernetes. + +## Relevant files and directory structure + - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. + - `common-images`: Static images. Those images can derive from base images can be also used as base images in CloudHarness apps and tasks. + +## Base images and common images + +The main difference between the base images and common images is that base images are built in the root context, while +common images are built in a local context. +So, base images are general purpose and are mainly used to provide access to custom libraries, while common images can have +a specific purpose (e.g. enable widely used libraries for tasks). + + + + diff --git a/blueprint/infrastructure/base-images/README.md b/blueprint/infrastructure/base-images/README.md new file mode 100644 index 00000000..6bd42adc --- /dev/null +++ b/blueprint/infrastructure/base-images/README.md @@ -0,0 +1,3 @@ +# Base images + +Here we find base images to inherit from our applications \ No newline at end of file diff --git a/blueprint/infrastructure/common-images/README.md b/blueprint/infrastructure/common-images/README.md new file mode 100644 index 00000000..a80ff6bb --- /dev/null +++ b/blueprint/infrastructure/common-images/README.md @@ -0,0 +1,3 @@ +#Common images + +Here we find common images reused in tasks and applications \ No newline at end of file diff --git a/deployment.yaml b/deployment.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/requirements.txt b/requirements.txt index 1c3cc80d..2a95221e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ --e utilities/cloudharness-deploy -pyaml +-e utilities +oyaml diff --git a/utilities/cloudharness_utilities.egg-info/PKG-INFO b/utilities/cloudharness_utilities.egg-info/PKG-INFO deleted file mode 100644 index 048dcc65..00000000 --- a/utilities/cloudharness_utilities.egg-info/PKG-INFO +++ /dev/null @@ -1,12 +0,0 @@ -Metadata-Version: 1.0 -Name: cloudharness-utilities -Version: 0.1.0 -Summary: CloudHarness deploy utilities library -Home-page: UNKNOWN -Author: UNKNOWN -Author-email: cloudharness@metacell.us -License: UNKNOWN -Description: MetaCell Neuroscience Platform - CloudHarness deploy library - -Keywords: Cloud,Kubernetes,Helm,Deploy -Platform: UNKNOWN diff --git a/utilities/cloudharness_utilities.egg-info/SOURCES.txt b/utilities/cloudharness_utilities.egg-info/SOURCES.txt deleted file mode 100644 index d185c324..00000000 --- a/utilities/cloudharness_utilities.egg-info/SOURCES.txt +++ /dev/null @@ -1,39 +0,0 @@ -MANIFEST.in -README.md -harness-application -harness-codefresh -harness-deployment -harness-generate -setup.py -cloudharness_utilities/__init__.py -cloudharness_utilities/build.py -cloudharness_utilities/codefresh.py -cloudharness_utilities/constants.py -cloudharness_utilities/helm.py -cloudharness_utilities/openapi.py -cloudharness_utilities/utils.py -cloudharness_utilities.egg-info/PKG-INFO -cloudharness_utilities.egg-info/SOURCES.txt -cloudharness_utilities.egg-info/dependency_links.txt -cloudharness_utilities.egg-info/requires.txt -cloudharness_utilities.egg-info/top_level.txt -cloudharness_utilities/application-template/api/config.json -cloudharness_utilities/application-template/api/samples.yaml -cloudharness_utilities/application-template/deploy/values.yaml -cloudharness_utilities/deployment-configuration/README.md -cloudharness_utilities/deployment-configuration/codefresh-build-template.yaml -cloudharness_utilities/deployment-configuration/codefresh-template.yaml -cloudharness_utilities/deployment-configuration/value-template.yaml -cloudharness_utilities/deployment-configuration/values-template.yaml -cloudharness_utilities/deployment-configuration/helm/.helmignore -cloudharness_utilities/deployment-configuration/helm/Chart.yaml -cloudharness_utilities/deployment-configuration/helm/README.md -cloudharness_utilities/deployment-configuration/helm/values.yaml -cloudharness_utilities/deployment-configuration/helm/templates/NOTES.txt -cloudharness_utilities/deployment-configuration/helm/templates/_helpers.tpl -cloudharness_utilities/deployment-configuration/helm/templates/auto-deployments.yaml -cloudharness_utilities/deployment-configuration/helm/templates/auto-gatekeepers.yaml -cloudharness_utilities/deployment-configuration/helm/templates/auto-services.yaml -cloudharness_utilities/deployment-configuration/helm/templates/ingress.yaml -cloudharness_utilities/deployment-configuration/helm/templates/secrets.yaml -cloudharness_utilities/deployment-configuration/helm/templates/certs/letsencrypt.yaml \ No newline at end of file diff --git a/utilities/cloudharness_utilities.egg-info/dependency_links.txt b/utilities/cloudharness_utilities.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/utilities/cloudharness_utilities.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/utilities/cloudharness_utilities.egg-info/requires.txt b/utilities/cloudharness_utilities.egg-info/requires.txt deleted file mode 100644 index 243869a0..00000000 --- a/utilities/cloudharness_utilities.egg-info/requires.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyaml -docker diff --git a/utilities/cloudharness_utilities.egg-info/top_level.txt b/utilities/cloudharness_utilities.egg-info/top_level.txt deleted file mode 100644 index 8612b548..00000000 --- a/utilities/cloudharness_utilities.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -cloudharness_utilities diff --git a/utilities/cloudharness_utilities/codefresh.py b/utilities/cloudharness_utilities/codefresh.py index b6ada5cb..5fa7769d 100644 --- a/utilities/cloudharness_utilities/codefresh.py +++ b/utilities/cloudharness_utilities/codefresh.py @@ -1,48 +1,65 @@ import os -import pyaml -import yaml +import oyaml as yaml import logging from .constants import HERE, BUILD_STEP_BASE, BUILD_STEP_STATIC, BUILD_STEP_PARALLEL, BUILD_STEP_INSTALL, \ CODEFRESH_REGISTRY, K8S_IMAGE_EXCLUDE, CODEFRESH_PATH, CODEFRESH_BUILD_PATH, \ - CODEFRESH_TEMPLATE_PATH, APPS_PATH, STATIC_IMAGES_PATH, BASE_IMAGES_PATH + CODEFRESH_TEMPLATE_PATH, APPS_PATH, STATIC_IMAGES_PATH, BASE_IMAGES_PATH, DEPLOYMENT_PATH from .helm import collect_helm_values from .utils import find_dockerfiles_paths, image_name_from_docker_path, \ get_image_name, get_template, merge_to_yaml_file logging.getLogger().setLevel(logging.INFO) -def create_codefresh_deployment_scripts(deployment_root, tag="${{CF_REVISION}}", codefresh_path=CODEFRESH_PATH): +CLOUD_HARNESS_PATH = 'cloud-harness' + + +def create_codefresh_deployment_scripts(deployment_root_path, tag="${{CF_REVISION}}", codefresh_path=CODEFRESH_PATH): """ Entry point to create deployment scripts for codefresh: codefresh.yaml and helm chart """ - codefresh = get_template(CODEFRESH_TEMPLATE_PATH) + codefresh = get_template(os.path.join(deployment_root_path, CODEFRESH_TEMPLATE_PATH)) codefresh['steps'][BUILD_STEP_BASE]['steps'] = {} codefresh['steps'][BUILD_STEP_STATIC]['steps'] = {} codefresh['steps'][BUILD_STEP_PARALLEL]['steps'] = {} def codefresh_build_step_from_base_path(base_path, build_step, root_context=None): - abs_base_path = os.path.join(deployment_root, base_path) + abs_base_path = os.path.join(deployment_root_path, base_path) for dockerfile_path in find_dockerfiles_paths(abs_base_path): - app_relative_to_root = os.path.relpath(dockerfile_path, deployment_root) + app_relative_to_root = os.path.relpath(dockerfile_path, deployment_root_path) app_relative_to_base = os.path.relpath(dockerfile_path, abs_base_path) app_name = image_name_from_docker_path(app_relative_to_base) if app_name in K8S_IMAGE_EXCLUDE: continue - build = codefresh_app_build_spec(app_name, os.path.relpath(root_context, deployment_root) if root_context else app_relative_to_root, - dockerfile_path=os.path.join(os.path.relpath(dockerfile_path, root_context) if root_context else '', "Dockerfile")) + build = codefresh_app_build_spec(app_name, os.path.relpath(root_context, + deployment_root_path) if root_context else app_relative_to_root, + dockerfile_path=os.path.join( + os.path.relpath(dockerfile_path, root_context) if root_context else '', + "Dockerfile")) codefresh['steps'][build_step]['steps'][app_name] = build - codefresh_build_step_from_base_path(BASE_IMAGES_PATH, BUILD_STEP_BASE, root_context=deployment_root) + codefresh_build_step_from_base_path(BASE_IMAGES_PATH, BUILD_STEP_BASE, root_context=deployment_root_path) codefresh_build_step_from_base_path(STATIC_IMAGES_PATH, BUILD_STEP_STATIC) codefresh_build_step_from_base_path(APPS_PATH, BUILD_STEP_PARALLEL) - codefresh['steps'] = {k:step for k, step in codefresh['steps'].items() if 'type' not in step or step['type'] != 'parallel' or step['steps']} + if os.path.exists(os.path.join(deployment_root_path, CLOUD_HARNESS_PATH)): + logging.info('Create build steps for cloud-harness images') + codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, BASE_IMAGES_PATH), BUILD_STEP_BASE, + root_context=deployment_root_path) + codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, STATIC_IMAGES_PATH), BUILD_STEP_STATIC) + codefresh_build_step_from_base_path(os.path.join(CLOUD_HARNESS_PATH, APPS_PATH), BUILD_STEP_PARALLEL) + + codefresh['steps'] = {k: step for k, step in codefresh['steps'].items() if + 'type' not in step or step['type'] != 'parallel' or (step['steps'] if 'steps' in step else [])} - with open(codefresh_path, 'w') as f: - pyaml.dump(codefresh, f) + codefresh_abs_path = os.path.join(deployment_root_path, DEPLOYMENT_PATH, codefresh_path) + codefresh_dir = os.path.dirname(codefresh_abs_path) + if not os.path.exists(codefresh_dir): + os.makedirs(codefresh_dir) + with open(codefresh_abs_path, 'w') as f: + yaml.dump(codefresh, f) def codefresh_build_spec(**kwargs): @@ -60,7 +77,8 @@ def codefresh_build_spec(**kwargs): def codefresh_app_build_spec(app_name, app_path, dockerfile_path="Dockerfile"): logging.info('Generating build script for ' + app_name) title = app_name.capitalize().replace('-', ' ').replace('/', ' ').replace('.', ' ').strip() - build = codefresh_build_spec(image_name=get_image_name(app_name), title=title, working_directory= './' + app_path, dockerfile=dockerfile_path) + build = codefresh_build_spec(image_name=get_image_name(app_name), title=title, working_directory='./' + app_path, + dockerfile=dockerfile_path) specific_build_template_path = os.path.join(app_path, 'build.yaml') if os.path.exists(specific_build_template_path): diff --git a/utilities/cloudharness_utilities/constants.py b/utilities/cloudharness_utilities/constants.py index 28def2ae..d8f7482a 100644 --- a/utilities/cloudharness_utilities/constants.py +++ b/utilities/cloudharness_utilities/constants.py @@ -13,23 +13,24 @@ BASE_IMAGES_PATH = os.path.join(INFRASTRUCTURE_PATH, 'base-images') NEUTRAL_PATHS = ('src', 'tasks', 'server') APPS_PATH = 'applications' - +DEPLOYMENT_PATH = 'deployment' CODEFRESH_PATH = 'codefresh/codefresh.yaml' -CODEFRESH_BUILD_PATH = 'deployment-templates/codefresh-build-template.yaml' -CODEFRESH_TEMPLATE_PATH = 'deployment-templates/codefresh-template.yaml' +DEPLOYMENT_CONFIGURATION_PATH = 'deployment-configuration' + +CODEFRESH_BUILD_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/codefresh-build-template.yaml' +CODEFRESH_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/codefresh-template.yaml' CODEFRESH_REGISTRY = "r.cfcr.io/tarelli" VALUES_MANUAL_PATH = 'values.yaml' -VALUE_TEMPLATE_PATH = 'deployment-templates/value-template.yaml' +VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml' CH_BASE_IMAGES = {'cloudharness-base': 'python:3.7-alpine', 'cloudharness-base-debian': 'python:3'} K8S_IMAGE_EXCLUDE = ('accounts-keycloak-gatekeeper',) -BUILD_STEP_BASE = 'x1_build_base_image' -BUILD_STEP_STATIC = 'x2_static_build' -BUILD_STEP_PARALLEL = 'x3_parallel_build' -BUILD_STEP_INSTALL = 'x4_deployment' +BUILD_STEP_BASE = 'build_base_images' +BUILD_STEP_STATIC = 'build_static_images' +BUILD_STEP_PARALLEL = 'build_application_images' +BUILD_STEP_INSTALL = 'deployment' -DEPLOYMENT_CONFIGURATION_PATH = 'deployment-configuration' \ No newline at end of file diff --git a/utilities/cloudharness_utilities/deployment-configuration/codefresh-template.yaml b/utilities/cloudharness_utilities/deployment-configuration/codefresh-template.yaml index c89d07c5..c95419b5 100644 --- a/utilities/cloudharness_utilities/deployment-configuration/codefresh-template.yaml +++ b/utilities/cloudharness_utilities/deployment-configuration/codefresh-template.yaml @@ -5,23 +5,29 @@ stages: - deploy steps: main_clone: - title: Cloning main repository... + title: Clone main repository type: git-clone stage: prepare repo: '${{CF_REPO_OWNER}}/${{CF_REPO_NAME}}' revision: '${{CF_BRANCH}}' git: github - x1_build_base_image: + post_main_clone: + title: Post main clone + type: parallel + stage: prepare + build_base_images: + title: Build base images type: parallel stage: build steps: REPLACE_ME - x2_static_build: + build_static_images: + title: Build static images type: parallel stage: build steps: REPLACE_ME - x3_parallel_build: + build_application_images: type: parallel stage: build steps: @@ -29,15 +35,16 @@ steps: prepare_deployment: title: "Prepare helm chart" - image: r.cfcr.io/tarelli/cloudharness-deploy + image: python:3.7 stage: deploy - working_directory: ./deployment + working_directory: . commands: - - python deployment-generate.py + - pip install -r cloud-harness/requirements.txt + - harness-deployment . cloud-harness -t ${{CF_REVISION}} -d ${{DOMAIN}} - prepare_deployment_view2: + prepare_deployment_view: commands: - - 'helm template ./deployment/helm --debug -n '${{NAME}} + - 'helm template ./deployment/helm --debug -n ${{NAME}}' environment: - ACTION=auth - KUBE_CONTEXT=${{NAME}} @@ -45,7 +52,7 @@ steps: stage: prepare title: 'View helm chart' - x4_deployment: + deployment: stage: deploy image: 'codefresh/cfstep-helm:2.16.1' title: Installing chart @@ -59,9 +66,4 @@ steps: - HELM_REPO_USE_HTTP=false - HELM_REPO_CONTEXT_PATH= - TIMEOUT=600 - - VALUESFILE_values=./deployment/helm/values.yaml - - when: - branch: - only: - - '${{CF_BRANCH}}' \ No newline at end of file + - VALUESFILE_values=./deployment/helm/values.yaml \ No newline at end of file diff --git a/utilities/cloudharness_utilities/utils.py b/utilities/cloudharness_utilities/utils.py index 38f47389..4c2b2dcc 100644 --- a/utilities/cloudharness_utilities/utils.py +++ b/utilities/cloudharness_utilities/utils.py @@ -2,8 +2,7 @@ import subprocess import os import collections -import yaml -import pyaml +import oyaml as yaml import shutil import logging @@ -110,7 +109,7 @@ def merge_to_yaml_file(content_src, fdest): if not os.path.exists(os.path.dirname(fdest)): os.makedirs(os.path.dirname(fdest)) with open(fdest, "w") as f: - pyaml.dump(merged, f) + yaml.dump(merged, f) return merged diff --git a/utilities/harness-codefresh b/utilities/harness-codefresh index 8a95a89c..37e6425c 100644 --- a/utilities/harness-codefresh +++ b/utilities/harness-codefresh @@ -15,13 +15,14 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description='Walk filesystem inside ./applications and define build and deploy scripts.') - parser.add_argument('paths', metavar='paths', default=ROOT, type=str, nargs='*', - help='Base paths to start looking for applications. The paths will be processed in the given order.') + parser.add_argument('path', metavar='path', default=ROOT, type=str, + help='Base path of the application.') args, unknown = parser.parse_known_args(sys.argv[1:]) + root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs(args.path) else args.path if unknown: print('There are unknown args. Make sure to call the script with the accepted args. Try --help') print(f'unknown: {unknown}') else: - create_codefresh_deployment_scripts(args.paths) + create_codefresh_deployment_scripts(root_path) diff --git a/utilities/harness-deployment b/utilities/harness-deployment index 398e7bda..9364088a 100644 --- a/utilities/harness-deployment +++ b/utilities/harness-deployment @@ -16,7 +16,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description='Walk filesystem inside ./applications and define build and deploy scripts.') - parser.add_argument('paths', metavar='paths', default=ROOT, type=str, nargs='*', + parser.add_argument('paths', metavar='paths', default=[ROOT], type=str, nargs='*', help='Base paths to start looking for applications. The paths will be processed in the given order.') parser.add_argument('-i', '--image', dest='image', action="append", default=[], diff --git a/utilities/requirements.txt b/utilities/requirements.txt index cd8adc79..c283be7e 100644 --- a/utilities/requirements.txt +++ b/utilities/requirements.txt @@ -1,2 +1,2 @@ -pyaml +oyaml docker \ No newline at end of file diff --git a/utilities/setup.py b/utilities/setup.py index 1bc8f265..c4a70ed6 100644 --- a/utilities/setup.py +++ b/utilities/setup.py @@ -21,7 +21,7 @@ # http://pypi.python.org/pypi/setuptools REQUIREMENTS = [ - 'pyaml', + 'oyaml', 'docker' ]