diff --git a/application-templates/base/.dockerignore b/application-templates/base/.dockerignore new file mode 100644 index 00000000..827cf7ca --- /dev/null +++ b/application-templates/base/.dockerignore @@ -0,0 +1,73 @@ +.travis.yaml +.openapi-generator-ignore +README.md +tox.ini +git_push.sh +test-requirements.txt + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +venv/ +.python-version + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints + +.git \ No newline at end of file diff --git a/application-templates/django-app/.dockerignore b/application-templates/django-app/.dockerignore index f06235c4..f2e3d311 100644 --- a/application-templates/django-app/.dockerignore +++ b/application-templates/django-app/.dockerignore @@ -1,2 +1,3 @@ node_modules dist +.git \ No newline at end of file diff --git a/application-templates/flask-server/backend/.dockerignore b/application-templates/flask-server/backend/.dockerignore index a05d73b5..827cf7ca 100644 --- a/application-templates/flask-server/backend/.dockerignore +++ b/application-templates/flask-server/backend/.dockerignore @@ -69,3 +69,5 @@ target/ #Ipython Notebook .ipynb_checkpoints + +.git \ No newline at end of file diff --git a/application-templates/webapp/.dockerignore b/application-templates/webapp/.dockerignore index f06235c4..827cf7ca 100644 --- a/application-templates/webapp/.dockerignore +++ b/application-templates/webapp/.dockerignore @@ -1,2 +1,73 @@ -node_modules -dist +.travis.yaml +.openapi-generator-ignore +README.md +tox.ini +git_push.sh +test-requirements.txt + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +venv/ +.python-version + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints + +.git \ No newline at end of file diff --git a/docs/applications/README.md b/docs/applications/README.md index b8758632..82fe472a 100644 --- a/docs/applications/README.md +++ b/docs/applications/README.md @@ -102,6 +102,7 @@ The most important configuration entries are the following: - `hard`: hard dependencies mean that they are required for this application to work properly - `soft`: the application will function for most of its functionality without this dependency - `build`: the images declared as build dependencies can be referred as dependency in the Dockerfile + - `git`: specify repos to be cloned before the container build - `database`: automatically generates a preconfigured database deployment for this application - `auto`: if true, turns on the database deployment functionality - `type`: one from `postgres` (default), `mongo`, `neo4j` diff --git a/docs/applications/dependencies.md b/docs/applications/dependencies.md new file mode 100644 index 00000000..e6d1a5b7 --- /dev/null +++ b/docs/applications/dependencies.md @@ -0,0 +1,91 @@ +# Application dependencies + +Application dependencies can be specified in the main application configuration +file (deploy/values.yaml), in the `harness` section. + +Example: +```yaml +harness: + dependencies: + build: + - cloudharness-base + soft: + - app1 + hard: + - accounts + git: + - url: https://github.com/a/b.git + branch_tag: master +``` + +## Build dependencies + +Build dependencies specify which images must be built before the current one. +Currently only base images and common images can be used as a build dependency. + +See also [base and common images documentation](../base-common-images.md). + +## Soft dependencies + +Soft dependencies specify other applications (from your app or cloudharness) that +must be included in the deployment together with your application, +but are not a prerequisite for the application to bootstrap and serve basic functionality. + +Soft dependencies are implicitly chained: if *A1* depends on *A2* and *A2* depends on *A3*, +all *A1*, *A2*, *A3* are included in the deployment if A1 is requested (say with +`harness-deployment ... -i A1`). + +## Hard dependencies + +Hard dependencies work similarly to soft dependencies but they are required for the +application declaring the dependency to start and provide even basic functionality. + +With a hard dependency, we are allowed to assume that the other application exists in the +configuration and during the runtime. + +Note that Cloud Harness does not guarantee the the other application starts before the +application declaring the dependency because that's how Kubernetes works. The application +is supposed to crash in the absence of its dependency and Kubernetes will start the crash +loop until both applications are settled. + +## Git (repository) dependencies + +Git dependencies allow us to build and deploy applications that are defined in another repository. +This functionality is an alternative to the otherwise monorepo-centric view of CloudHarness-based +applications. + +The repository is cloned before the build within skaffold build and the CI/CD inside the +`dependencies` directory at the same level of the Dockerfile. + +Hence, in the Dockerfile we are allowed to `COPY` or `ADD` the repository. + +For instance, given the following configuration: +```yaml +harness: + dependencies: + git: + - url: https://github.com/a/b.git + branch_tag: master + - url: https://github.com/c/d.git + branch_tag: v1.0.0 + path: myrepo +``` + +The directory structure will be as following: +``` +Dockerfile +dependencies + b.git + myrepo + .dockerignore +``` + +Hence, inside the Dockerfile we expect to see something like + +```dockerfile +COPY dependencies . +COPY dependencies/b.git/src . +COPY dependencies/myrepo . +``` + +> Note that Cloud Harness does not add the COPY/ADD statements to the Dockerfile \ No newline at end of file diff --git a/docs/applications/development/backend-development.md b/docs/applications/development/backend-development.md index 34d044bb..169f495e 100644 --- a/docs/applications/development/backend-development.md +++ b/docs/applications/development/backend-development.md @@ -55,7 +55,9 @@ harness: Every image defined as a base image or a common image can be used as a build dependency. -For more details about how to define your custom image and the available images, see [here](../../base-common-images.md) +For more details about how to define your custom image and the available images, see [here](../../base-common-images.md). + +For more info about dependencies, see [here](../dependencies.md) ## Use the CloudHarness runtime Python library diff --git a/docs/base-common-images.md b/docs/base-common-images.md index 314995f8..29a5e2bf 100644 --- a/docs/base-common-images.md +++ b/docs/base-common-images.md @@ -15,9 +15,9 @@ a specific purpose (e.g. enable widely used application stacks to inherit from). After generating the codeChange the Dockerfile in order to inherit from the main Docker image need to: -1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located +1. Add the image as a [build dependency](applications/dependencies.md) to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located -``` +```yaml harness: dependencies: build: diff --git a/docs/model/ApplicationDependenciesConfig.md b/docs/model/ApplicationDependenciesConfig.md index 8d3ba917..a59b2cec 100644 --- a/docs/model/ApplicationDependenciesConfig.md +++ b/docs/model/ApplicationDependenciesConfig.md @@ -11,6 +11,7 @@ Key | Input Type | Accessed Type | Description | Notes **[hard](#hard)** | list, tuple, | tuple, | Hard dependencies indicate that the application may not start without these other applications. | [optional] **[soft](#soft)** | list, tuple, | tuple, | Soft dependencies indicate that the application will work partially without these other applications. | [optional] **[build](#build)** | list, tuple, | tuple, | Hard dependencies indicate that the application Docker image build requires these base/common images | [optional] +**[git](#git)** | list, tuple, | tuple, | | [optional] **any_string_name** | dict, frozendict.frozendict, str, date, datetime, int, float, bool, decimal.Decimal, None, list, tuple, bytes, io.FileIO, io.BufferedReader | frozendict.frozendict, str, BoolClass, decimal.Decimal, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional] # hard @@ -55,5 +56,17 @@ Class Name | Input Type | Accessed Type | Description | Notes ------------- | ------------- | ------------- | ------------- | ------------- items | str, | str, | | +# git + +## Model Type Info +Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- +list, tuple, | tuple, | | + +### Tuple Items +Class Name | Input Type | Accessed Type | Description | Notes +------------- | ------------- | ------------- | ------------- | ------------- +[**GitDependencyConfig**](GitDependencyConfig.md) | [**GitDependencyConfig**](GitDependencyConfig.md) | [**GitDependencyConfig**](GitDependencyConfig.md) | | + [[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md) diff --git a/docs/model/GitDependencyConfig.md b/docs/model/GitDependencyConfig.md new file mode 100644 index 00000000..5faef206 --- /dev/null +++ b/docs/model/GitDependencyConfig.md @@ -0,0 +1,19 @@ +# cloudharness_model.model.git_dependency_config.GitDependencyConfig + +Defines a git repo to be cloned inside the application path + +## Model Type Info +Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- +dict, frozendict.frozendict, | frozendict.frozendict, | Defines a git repo to be cloned inside the application path | + +### Dictionary Keys +Key | Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- | ------------- +**branch_tag** | str, | str, | | +**url** | str, | str, | | +**path** | str, | str, | Defines the path where the repo is cloned. default: /git | [optional] +**any_string_name** | dict, frozendict.frozendict, str, date, datetime, int, float, bool, decimal.Decimal, None, list, tuple, bytes, io.FileIO, io.BufferedReader | frozendict.frozendict, str, BoolClass, decimal.Decimal, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional] + +[[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md) + diff --git a/libraries/cloudharness-utils/cloudharness_utils/constants.py b/libraries/cloudharness-utils/cloudharness_utils/constants.py index 4b42761a..e2a6a48b 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/constants.py +++ b/libraries/cloudharness-utils/cloudharness_utils/constants.py @@ -43,7 +43,7 @@ CD_E2E_TEST_STEP = 'tests_e2e' CD_STEP_PUBLISH = 'publish' BUILD_FILENAMES = ('node_modules',) - +CD_BUILD_STEP_DEPENDENCIES = 'post_main_clone' E2E_TESTS_DIRNAME = 'e2e' API_TESTS_DIRNAME = 'api' diff --git a/libraries/models/api/openapi.yaml b/libraries/models/api/openapi.yaml index 96a3e480..8b961031 100644 --- a/libraries/models/api/openapi.yaml +++ b/libraries/models/api/openapi.yaml @@ -68,6 +68,11 @@ components: type: array items: type: string + git: + description: '' + type: array + items: + $ref: '#/components/schemas/GitDependencyConfig' DeploymentResourcesConf: description: '' type: object @@ -849,3 +854,22 @@ components: example: quota-ws-max: 5 quota-storage-max: 1G + GitDependencyConfig: + title: Root Type for GitDependencyConfig + description: Defines a git repo to be cloned inside the application path + required: + - branch_tag + - url + type: object + properties: + url: + type: string + branch_tag: + type: string + path: + description: 'Defines the path where the repo is cloned. default: /git' + type: string + example: + url: 'https://github.com/MetaCell/nwb-explorer.git' + branch_tag: master + path: /git diff --git a/libraries/models/cloudharness_model/models/__init__.py b/libraries/models/cloudharness_model/models/__init__.py index 78d8f9fc..dbfb1175 100644 --- a/libraries/models/cloudharness_model/models/__init__.py +++ b/libraries/models/cloudharness_model/models/__init__.py @@ -25,6 +25,7 @@ from cloudharness_model.models.deployment_volume_spec_all_of import DeploymentVolumeSpecAllOf from cloudharness_model.models.e2_e_tests_config import E2ETestsConfig from cloudharness_model.models.file_resources_config import FileResourcesConfig +from cloudharness_model.models.git_dependency_config import GitDependencyConfig from cloudharness_model.models.harness_main_config import HarnessMainConfig from cloudharness_model.models.ingress_config import IngressConfig from cloudharness_model.models.ingress_config_all_of import IngressConfigAllOf diff --git a/libraries/models/cloudharness_model/models/application_dependencies_config.py b/libraries/models/cloudharness_model/models/application_dependencies_config.py index d4330359..7f4df0ba 100644 --- a/libraries/models/cloudharness_model/models/application_dependencies_config.py +++ b/libraries/models/cloudharness_model/models/application_dependencies_config.py @@ -6,8 +6,10 @@ from typing import List, Dict # noqa: F401 from cloudharness_model.models.base_model_ import Model +from cloudharness_model.models.git_dependency_config import GitDependencyConfig from cloudharness_model import util +from cloudharness_model.models.git_dependency_config import GitDependencyConfig # noqa: E501 class ApplicationDependenciesConfig(Model): """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). @@ -15,7 +17,7 @@ class ApplicationDependenciesConfig(Model): Do not edit the class manually. """ - def __init__(self, hard=None, soft=None, build=None): # noqa: E501 + def __init__(self, hard=None, soft=None, build=None, git=None): # noqa: E501 """ApplicationDependenciesConfig - a model defined in OpenAPI :param hard: The hard of this ApplicationDependenciesConfig. # noqa: E501 @@ -24,22 +26,27 @@ def __init__(self, hard=None, soft=None, build=None): # noqa: E501 :type soft: List[str] :param build: The build of this ApplicationDependenciesConfig. # noqa: E501 :type build: List[str] + :param git: The git of this ApplicationDependenciesConfig. # noqa: E501 + :type git: List[GitDependencyConfig] """ self.openapi_types = { 'hard': List[str], 'soft': List[str], - 'build': List[str] + 'build': List[str], + 'git': List[GitDependencyConfig] } self.attribute_map = { 'hard': 'hard', 'soft': 'soft', - 'build': 'build' + 'build': 'build', + 'git': 'git' } self._hard = hard self._soft = soft self._build = build + self._git = git @classmethod def from_dict(cls, dikt) -> 'ApplicationDependenciesConfig': @@ -120,3 +127,26 @@ def build(self, build): """ self._build = build + + @property + def git(self): + """Gets the git of this ApplicationDependenciesConfig. + + # noqa: E501 + + :return: The git of this ApplicationDependenciesConfig. + :rtype: List[GitDependencyConfig] + """ + return self._git + + @git.setter + def git(self, git): + """Sets the git of this ApplicationDependenciesConfig. + + # noqa: E501 + + :param git: The git of this ApplicationDependenciesConfig. + :type git: List[GitDependencyConfig] + """ + + self._git = git diff --git a/libraries/models/cloudharness_model/models/git_dependency_config.py b/libraries/models/cloudharness_model/models/git_dependency_config.py new file mode 100644 index 00000000..b2a17690 --- /dev/null +++ b/libraries/models/cloudharness_model/models/git_dependency_config.py @@ -0,0 +1,122 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from cloudharness_model.models.base_model_ import Model +from cloudharness_model import util + + +class GitDependencyConfig(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, url=None, branch_tag=None, path=None): # noqa: E501 + """GitDependencyConfig - a model defined in OpenAPI + + :param url: The url of this GitDependencyConfig. # noqa: E501 + :type url: str + :param branch_tag: The branch_tag of this GitDependencyConfig. # noqa: E501 + :type branch_tag: str + :param path: The path of this GitDependencyConfig. # noqa: E501 + :type path: str + """ + self.openapi_types = { + 'url': str, + 'branch_tag': str, + 'path': str + } + + self.attribute_map = { + 'url': 'url', + 'branch_tag': 'branch_tag', + 'path': 'path' + } + + self._url = url + self._branch_tag = branch_tag + self._path = path + + @classmethod + def from_dict(cls, dikt) -> 'GitDependencyConfig': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The GitDependencyConfig of this GitDependencyConfig. # noqa: E501 + :rtype: GitDependencyConfig + """ + return util.deserialize_model(dikt, cls) + + @property + def url(self): + """Gets the url of this GitDependencyConfig. + + + :return: The url of this GitDependencyConfig. + :rtype: str + """ + return self._url + + @url.setter + def url(self, url): + """Sets the url of this GitDependencyConfig. + + + :param url: The url of this GitDependencyConfig. + :type url: str + """ + if url is None: + raise ValueError("Invalid value for `url`, must not be `None`") # noqa: E501 + + self._url = url + + @property + def branch_tag(self): + """Gets the branch_tag of this GitDependencyConfig. + + + :return: The branch_tag of this GitDependencyConfig. + :rtype: str + """ + return self._branch_tag + + @branch_tag.setter + def branch_tag(self, branch_tag): + """Sets the branch_tag of this GitDependencyConfig. + + + :param branch_tag: The branch_tag of this GitDependencyConfig. + :type branch_tag: str + """ + if branch_tag is None: + raise ValueError("Invalid value for `branch_tag`, must not be `None`") # noqa: E501 + + self._branch_tag = branch_tag + + @property + def path(self): + """Gets the path of this GitDependencyConfig. + + Defines the path where the repo is cloned. default: /git # noqa: E501 + + :return: The path of this GitDependencyConfig. + :rtype: str + """ + return self._path + + @path.setter + def path(self, path): + """Sets the path of this GitDependencyConfig. + + Defines the path where the repo is cloned. default: /git # noqa: E501 + + :param path: The path of this GitDependencyConfig. + :type path: str + """ + + self._path = path diff --git a/tools/clone.sh b/tools/clone.sh new file mode 100644 index 00000000..246c4d89 --- /dev/null +++ b/tools/clone.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +REPOSRC=$2 +LOCALREPO=$3 +BRANCH=$1 + +# We do it this way so that we can abstract if from just git later on +LOCALREPO_VC_DIR=$LOCALREPO/.git + +if [ ! -d $LOCALREPO_VC_DIR ] +then + git clone --branch $BRANCH $REPOSRC $LOCALREPO +else + cd $LOCALREPO + git pull origin $BRANCH +fi + +# End \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 6432233b..d6b699a5 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -1,5 +1,6 @@ import os from os.path import join, relpath, exists, dirname +from cloudharness_model.models.git_dependency_config import GitDependencyConfig import requests import logging from cloudharness_model.models.api_tests_config import ApiTestsConfig @@ -33,6 +34,27 @@ def literal_presenter(dumper, data): yaml.add_representer(str, literal_presenter) +def get_main_domain(url): + try: + url = url.split("//")[1].split("/")[0] + if "gitlab" in url: + return "gitlab" + if "bitbucket" in url: + return "bitbucket" + return "github" + except: + return "${{ DEFAULT_REPO }}" + +def clone_step_spec(conf: GitDependencyConfig, context_path: str): + return { + "title": f"Cloning {os.path.basename(conf.url)} repository...", + "type": "git-clone", + "repo": conf.url, + "revision": conf.branch_tag, + "working_directory": join(context_path, "dependencies", conf.path or os.path.basename(conf.url)), + "git": get_main_domain(conf.url) # Cannot really tell what's the git config name, usually the name of the repo + } + def write_env_file(helm_values: HarnessMainConfig, filename, registry_secret=None): env = {} logging.info("Create env file with image info %s", filename) @@ -152,6 +174,10 @@ def codefresh_steps_from_base_path(base_path, build_step, fixed_context=None, in # Skip excluded apps continue + if app_config and app_config.dependencies and app_config.dependencies.git: + for dep in app_config.dependencies.git: + steps[CD_BUILD_STEP_DEPENDENCIES]['steps'].append(clone_step_spec(dep, base_path)) + build = None if build_step in steps: build = codefresh_app_build_spec( diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index c0de5764..bd2f4da4 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -4,7 +4,7 @@ import time from os.path import join, relpath, basename, exists, abspath -from cloudharness_model import ApplicationTestConfig, HarnessMainConfig +from cloudharness_model import ApplicationTestConfig, HarnessMainConfig, GitDependencyConfig from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ BASE_IMAGES_PATH, STATIC_IMAGES_PATH @@ -57,7 +57,8 @@ def build_artifact(image_name, context_path, requirements=None, dockerfile_path= def process_build_dockerfile(dockerfile_path, root_path, global_context=False, requirements=None, app_name=None): if app_name is None: app_name = app_name_from_path(basename(dockerfile_path)) - if app_name in helm_values[KEY_TASK_IMAGES] or app_name.replace("-", "_") in helm_values.apps: + app_key = app_name.replace("-", "_") + if app_name in helm_values[KEY_TASK_IMAGES] or app_key in helm_values.apps: context_path = relpath_if(root_path, output_path) if global_context else relpath_if(dockerfile_path, output_path) builds[app_name] = context_path @@ -68,6 +69,10 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r dockerfile_path=relpath(dockerfile_path, output_path), requirements=requirements or guess_build_dependencies_from_dockerfile(dockerfile_path) ) + if app_key in helm_values.apps and helm_values.apps[app_key].harness.dependencies and helm_values.apps[app_key].harness.dependencies.git: + artifacts[app_name]['hooks'] = { + 'before': [git_clone_hook(conf, context_path) for conf in helm_values.apps[app_key].harness.dependencies.git] + } for root_path in root_paths: skaffold_conf = dict_merge(skaffold_conf, get_template( @@ -189,6 +194,16 @@ def identify_unicorn_based_main(candidates): output_path, 'skaffold.yaml')) return skaffold_conf +def git_clone_hook(conf: GitDependencyConfig, context_path: str): + return { + 'command': [ + 'sh', + 'tools/clone.sh', + conf.branch_tag, + conf.url, + join(context_path, "dependencies", conf.path or os.path.basename(conf.url)) + ] + } def create_vscode_debug_configuration(root_paths, helm_values): logging.info( diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 983b6a55..9e2cbf9e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -59,7 +59,22 @@ def find_subdirs(base_path): def find_dockerfiles_paths(base_directory): - return find_file_paths(base_directory, 'Dockerfile') + all_dockerfiles = find_file_paths(base_directory, 'Dockerfile') + + # We want to remove all dockerfiles that are not in a git repository + # This will exclude the cloned dependencies and other repos cloned for convenience + dockerfiles_without_git = [] + + for dockerfile in all_dockerfiles: + directory = dockerfile + while not os.path.samefile(directory, base_directory): + if os.path.exists(os.path.join(directory, '.git')): + break + directory = os.path.dirname(directory) + else: + dockerfiles_without_git.append(dockerfile.replace(os.sep, "/")) + + return tuple(dockerfiles_without_git) def get_parent_app_name(app_relative_path): diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/dependencies/a/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/myapp/dependencies/a/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml index 434b042b..e386702a 100644 --- a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml +++ b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml @@ -6,6 +6,12 @@ harness: - legacy build: - cloudharness-flask + git: + - url: https://github.com/a/b.git + branch_tag: master + - url: https://github.com/c/d.git + branch_tag: v1.0.0 + path: myrepo test: unit: commands: diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index aee93d37..7e3abd4d 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -9,6 +9,9 @@ CLOUDHARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) BUILD_MERGE_DIR = "./build/test_deployment" +myapp_path = os.path.join(HERE, "resources/applications/myapp") +if not os.path.exists(os.path.join(myapp_path, "dependencies/a/.git")): + os.makedirs(os.path.join(myapp_path, "dependencies/a/.git")) def test_create_codefresh_configuration(): values = create_helm_chart( @@ -123,6 +126,8 @@ def test_create_codefresh_configuration(): assert len( tstep['commands']) == 2, "Unit test commands are not properly loaded from the unit test configuration file" assert tstep['commands'][0] == "tox", "Unit test commands are not properly loaded from the unit test configuration file" + + assert len(l1_steps[CD_BUILD_STEP_DEPENDENCIES]['steps']) == 3, "3 clone steps should be included as we have 2 dependencies from myapp, plus cloudharness" finally: shutil.rmtree(BUILD_MERGE_DIR) diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index 94998892..04cf4420 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -86,7 +86,8 @@ def test_create_skaffold_configuration(): a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/myapp') assert os.path.samefile(myapp_artifact['context'], join( RESOURCES, 'applications/myapp')) - + assert myapp_artifact['hooks']['before'], 'The hook for dependencies should be included' + assert len(myapp_artifact['hooks']['before']) == 2, 'The hook for dependencies should include 2 clone commands' accounts_artifact = next( a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/accounts') assert os.path.samefile(accounts_artifact['context'], '/tmp/build/applications/accounts') diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 507f75b9..cbd813c1 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -64,4 +64,14 @@ def test_guess_build_dependencies_from_dockerfile(): def test_check_docker_manifest_exists(): assert check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "latest") assert not check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "RANDOM_TAG") - \ No newline at end of file + +def test_find_dockerfile_paths(): + + myapp_path = os.path.join(HERE, "resources/applications/myapp") + if not os.path.exists(os.path.join(myapp_path, "dependencies/a/.git")): + os.makedirs(os.path.join(myapp_path, "dependencies/a/.git")) + + dockerfiles = find_dockerfiles_paths(myapp_path) + assert len(dockerfiles) == 2 + assert next(d for d in dockerfiles if d.endswith("myapp")), "Must find the Dockerfile in the root directory" + assert next(d for d in dockerfiles if d.endswith("myapp/tasks/mytask")), "Must find the Dockerfile in the tasks directory" \ No newline at end of file