See api documentation here
diff --git a/applications/samples/frontend/src/components/Version.tsx b/applications/samples/frontend/src/components/Version.tsx new file mode 100644 index 00000000..2b5f5eb3 --- /dev/null +++ b/applications/samples/frontend/src/components/Version.tsx @@ -0,0 +1,19 @@ +import React, { useState, useEffect } from 'react'; + + + +const Version = () => { + const [result, setResult] = useState(null); + useEffect(() => { + fetch("/proxy/common/api/version", { + headers: { + 'Accept': 'application/json' + } + }).then(r => r.json().then(j => setResult(j)), () => setResult({ data: "API error" })); + }, []); + + + return result ?Tag: { result?.tag } - Build: {result?.build}
:Backend did not answer
+} + +export default Version; \ No newline at end of file diff --git a/applications/samples/frontend/webpack.config.js b/applications/samples/frontend/webpack.config.js index ad5ee556..69f80cb9 100644 --- a/applications/samples/frontend/webpack.config.js +++ b/applications/samples/frontend/webpack.config.js @@ -29,7 +29,7 @@ module.exports = function webpacking(envVariables) { const output = { path: path.resolve(__dirname, "dist"), - filename: "[name].[contenthash].js", + filename: "js/[name].[contenthash].js", publicPath: "/" }; diff --git a/docs/applications/README.md b/docs/applications/README.md index b8758632..82fe472a 100644 --- a/docs/applications/README.md +++ b/docs/applications/README.md @@ -102,6 +102,7 @@ The most important configuration entries are the following: - `hard`: hard dependencies mean that they are required for this application to work properly - `soft`: the application will function for most of its functionality without this dependency - `build`: the images declared as build dependencies can be referred as dependency in the Dockerfile + - `git`: specify repos to be cloned before the container build - `database`: automatically generates a preconfigured database deployment for this application - `auto`: if true, turns on the database deployment functionality - `type`: one from `postgres` (default), `mongo`, `neo4j` diff --git a/docs/applications/dependencies.md b/docs/applications/dependencies.md new file mode 100644 index 00000000..e6d1a5b7 --- /dev/null +++ b/docs/applications/dependencies.md @@ -0,0 +1,91 @@ +# Application dependencies + +Application dependencies can be specified in the main application configuration +file (deploy/values.yaml), in the `harness` section. + +Example: +```yaml +harness: + dependencies: + build: + - cloudharness-base + soft: + - app1 + hard: + - accounts + git: + - url: https://github.com/a/b.git + branch_tag: master +``` + +## Build dependencies + +Build dependencies specify which images must be built before the current one. +Currently only base images and common images can be used as a build dependency. + +See also [base and common images documentation](../base-common-images.md). + +## Soft dependencies + +Soft dependencies specify other applications (from your app or cloudharness) that +must be included in the deployment together with your application, +but are not a prerequisite for the application to bootstrap and serve basic functionality. + +Soft dependencies are implicitly chained: if *A1* depends on *A2* and *A2* depends on *A3*, +all *A1*, *A2*, *A3* are included in the deployment if A1 is requested (say with +`harness-deployment ... -i A1`). + +## Hard dependencies + +Hard dependencies work similarly to soft dependencies but they are required for the +application declaring the dependency to start and provide even basic functionality. + +With a hard dependency, we are allowed to assume that the other application exists in the +configuration and during the runtime. + +Note that Cloud Harness does not guarantee the the other application starts before the +application declaring the dependency because that's how Kubernetes works. The application +is supposed to crash in the absence of its dependency and Kubernetes will start the crash +loop until both applications are settled. + +## Git (repository) dependencies + +Git dependencies allow us to build and deploy applications that are defined in another repository. +This functionality is an alternative to the otherwise monorepo-centric view of CloudHarness-based +applications. + +The repository is cloned before the build within skaffold build and the CI/CD inside the +`dependencies` directory at the same level of the Dockerfile. + +Hence, in the Dockerfile we are allowed to `COPY` or `ADD` the repository. + +For instance, given the following configuration: +```yaml +harness: + dependencies: + git: + - url: https://github.com/a/b.git + branch_tag: master + - url: https://github.com/c/d.git + branch_tag: v1.0.0 + path: myrepo +``` + +The directory structure will be as following: +``` +Dockerfile +dependencies + b.git + myrepo + .dockerignore +``` + +Hence, inside the Dockerfile we expect to see something like + +```dockerfile +COPY dependencies . +COPY dependencies/b.git/src . +COPY dependencies/myrepo . +``` + +> Note that Cloud Harness does not add the COPY/ADD statements to the Dockerfile \ No newline at end of file diff --git a/docs/applications/development/backend-development.md b/docs/applications/development/backend-development.md index 34d044bb..169f495e 100644 --- a/docs/applications/development/backend-development.md +++ b/docs/applications/development/backend-development.md @@ -55,7 +55,9 @@ harness: Every image defined as a base image or a common image can be used as a build dependency. -For more details about how to define your custom image and the available images, see [here](../../base-common-images.md) +For more details about how to define your custom image and the available images, see [here](../../base-common-images.md). + +For more info about dependencies, see [here](../dependencies.md) ## Use the CloudHarness runtime Python library diff --git a/docs/base-common-images.md b/docs/base-common-images.md index f3484f48..6d0ac6bf 100644 --- a/docs/base-common-images.md +++ b/docs/base-common-images.md @@ -16,9 +16,9 @@ a specific purpose (e.g. enable widely used application stacks to inherit from). After generating the codeChange the Dockerfile in order to inherit from the main Docker image need to: -1. Add the image as a build dependency to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located +1. Add the image as a [build dependency](applications/dependencies.md) to the values.yaml file of your application. The name of the image corresponds to the directory name where the Dockerfile is located -``` +```yaml harness: dependencies: build: diff --git a/docs/common-api.md b/docs/common-api.md new file mode 100644 index 00000000..9d3b79dc --- /dev/null +++ b/docs/common-api.md @@ -0,0 +1,37 @@ +# Common API microservice + +The common microservice is intended to provide utility information about the +deployment and its configuration to the frontends. + +## Functionality +The main functionalities of the common microservice are: +- Information about the current version/build +- Accounts endpoint and configuration information +- Sentry endpoint + +## How to use it in your application + +First of all, have to configure your application deployment to include +the common microservice on the dependencies and used services. + +`myapp/deploy/values.yaml` +```yaml +harness: + ... + dependencies: + soft: + - common + ... + ... + use_services: + - name: common +``` + +The common api will be available at `/proxy/common/api` path from your app + +> the `use_services` sets up the reverse proxy in your app subdomain +> to avoid cross-origin requests from the frontend + +See a usage example [here](../applications/samples/frontend/src/components/Version.tsx). + + diff --git a/docs/dev.md b/docs/dev.md new file mode 100644 index 00000000..113e0c86 --- /dev/null +++ b/docs/dev.md @@ -0,0 +1,313 @@ +# CloudHarness, developer documentation + +This documentation is meant to be read by developers that needs to make modifications in CloudHarness. +The goal of this doc is to show how CloudHarness is internally built, the different parts of the code/files that are relative to specific features, and to provide a map to be able to modify or implement new features quickly. + +CloudHarness is a project that allows you to: quickly generate the code of your webapp, considering that it runs in the cloud with a micro-service architecture, and to easily connect all those micro-services together to finally build the final app. +Currently, the tools that CloudHarness can consider to build the final app are the following: + +* [OpenAPI](https://www.openapis.org/) for generating the model and API of your application (based on an OpenAPI specification), +* [KeyCloak](https://www.keycloak.org/) for the authentication, +* [Argo Workflow](https://argoproj.github.io/argo-workflows/) for orchestrating Kubernete workflows. You can consider Kunernete workflows with Argo as specific actions that needs to be executed in a isolated environement as it requires more resources or can take time. Usually they are started after a user action. +* [Kafka](https://kafka.apache.org/documentation/) to stream the events from each micro-services and notify listeners that a micro-service or a workflow finished its work, +* [Sentry](https://docs.sentry.io/) to report and log errors and run time exceptions, +* [JupyterHub](https://jupyter.org/hub) to provide jupyter notebooks access to a group of users, +* [Volume Manager](../applications/volumemanager/) to deal with external file system, +* [NFS Server](../applications/nfsserver/) to provide storage of file on an external NFS file system, +* [Kubernete](https://kubernetes.io/) is used to manage the auto-scaling, deployements, ... of micro-services on a cluster, +* [Code Fresh](https://codefresh.io/) for the remote build of the application, and it is configured to initiate a deployment on a remote Kubernete cluster, +* [Helm Chart](https://helm.sh/docs/topics/charts/) for the packaging of Kubernete resources to simplify the deployment of the application, +* [Skaffold](https://skaffold.dev/) to help deploying the packaged application in a Kubernete cluster. + +CloudHarness is made of two major parts: + +1. a command line interface (CLI) that helps bootstrapping the infrastructure for a dedicated tool or project, +2. a runtime which provides several helpers and already pre-coded services to handle the different micro-services. + +## The CloudHarness CLI + +The command line interface is used to generate various aspects of your webapp. +Basically, the CloudHarness CLI can generate (depending on your command line option): + +1. the skeleton (the various directory and stub files) for your webapp, depending on your needs, *e.g.:* a Django-based backend and a React-based frontend, +2. the base configuration files for Code Fresh: `codefresh-XXX.yaml` files, +3. the Helm Chart files for packaging your app, +4. the skaffold configuration file `skaffold.yaml`, +5. the copy of different pre-coded micro-services if required, *e.g.:* micro-service for authentication, based on KeyCloak, ... +6. the SSL certificate. + +The CloudHarness CLI project is located in [`tools/deployment-cli-tools`](../tools/deployment-cli-tools). +The source code of the project is located in [`tools/deployment-cli-tools/ch_cli_tools`](../tools/deployment-cli-tools/ch_cli_tools). +The code is organized around the idea that there is a module by artifact that can be generated: + +```bash +deployment-cli-tools +├── ch_cli_tools +│ ├── codefresh.py # Code Fresh configuration generation +│ ├── helm.py # Helm chart files generation +│ ├── __init__.py # Defines logging level and some global constants +│ ├── models.py # Currently empty file +│ ├── openapi.py # Generates the model and API part of your model (back and front) depending on an OpenAPI specification +│ ├── preprocessing.py # Provide some function to readapt/preprocess paths for the Helm generation +│ ├── scripts +│ │ ├── bootstrap.sh # Shell script for generating certificates for the app depending on the domain name +│ ├── skaffold.py # Skaffold configuration script generation +│ └── utils.py # Set of utils that are use to deal with directory/dict merging, path search, ... +├── harness-application # The main entry/script to create the base application and generate the base code from the OpenAPI specification(skeleton) +├── harness-deployment # The main entry/script to create the deployement for the application, based on some CloudHarness configuration files +├── harness-generate # The main entry/script to (re-)generate the base code for the frontend/backend from the OpenAPI specification, without crating folders for the application +└── tests/* # The tests folder for the CLI tools +``` + +### Generation of the base application skeleton + +The generation of the base application skeleton is obtain through the [`harness-application`](../tools/deployment-cli-tools/harness-application) command. +The command parses the type of application that needs to be generated. +If a new generator for a type of application needs to be defined, the main function of the script should be modified. + +The generation of the application is done in two times. +First the skeleton of the application is generated (the directories, basic files), then the code of REST application (server and client) is generated from the OpenAPI specification. +The following code fragment from the `harness-application` script shows how the skeleton is produced: + +```python +if "django-app" in args.templates and "webapp" not in templates: + templates = ["base", "webapp"] + templates + for template_name in templates: + if template_name == 'server': + with tempfile.TemporaryDirectory() as tmp_dirname: + copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) # <1> + merge_configuration_directories(app_path, tmp_dirname) + generate_server(app_path, tmp_dirname) + for base_path in (CH_ROOT, os.getcwd()): + template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) + if os.path.exists(template_path): + merge_configuration_directories(template_path, app_path) # <1> +``` + +First, if `django-app` is defined as a template for the application, and the `webapp` template is not set, then `base` and `webapp` are added to the list of templates. +Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists), as seen in `<1>`. +The templates for each type of application is described by the constant `APPLICATION_TEPLATE_PATH` and points to [`application-templates`](../application-templates/). +Based on the name of the template used for the application generation, the actual template with the same name is searched in this path, and copied/merged in the application target folder. +The constant, as well as many other constants, are located in [`cloudharness_utils.constants`](../libraries/cloudharness-utils/cloudharness_utils/constants.py). +This file is part of the CloudHarness runtime. +Other constants are located there as shown in the following code extract. + +```python +NODE_BUILD_IMAGE = 'node:8.16.1-alpine' +APPLICATION_TEMPLATE_PATH = 'application-templates' +# ... +APPS_PATH = 'applications' +DEPLOYMENT_PATH = 'deployment' +CODEFRESH_PATH = 'codefresh/codefresh.yaml' +# ... +CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10-alpine', 'cloudharness-base-debian': 'python:3.9.10'} +# ... +``` + +Those constants defines several aspects of CloudHarness. +For example, we can see there what base Docker image will be considered depending on what's configured for your application, where will be located the deployment files, from where the applications to generate/pick should be generated, where are located the templates for each kind of generation target, as well as where the configuration for codefresh should be looked for. + +Once the skeleton of the application is generated considering some templates, the code of the REST API is generated from the OpenAPI specification. +The generation relies on two functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. +Those functions are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. +This module and those functions use `openapi-generator-cli` to generate the code for the backend and/or the frontend. +With this generation, and depending on the templates used, some fine tuning or performed in the code/files generated. +For example, some placeholders are replaced depending on the name of the application, or depending on the module in which the application is generated. + +#### How to extend it? + +Here is some scenarios that would need to modify or impact this part of CloudHarness: + +**A new template for a directory/file skeleton needs to be added**. In this case, if a new template needs to be added, there is various operations that needs to be performed: + +1. a new template folder with the basic skeleton for the application needs to be created in [`applications-templates`](../application-templates/) with the name that the template should have as CLI argument, +2. modify the [`harness-application`](../tools/deployment-cli-tools/harness-application) script to include the new template, +3. add, if necessary, a new function in [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) to deal with the generation of the API REST code depending on your new template, +4. alter, if necessary, the configuration files that are generated in the [`harness-application`](../tools/deployment-cli-tools/harness-application) script. + +**Change/add the application base images**. In this case, if a new base image, or an existing base image should be modified, then the dictionnary located in [`constants.py`](../libraries/cloudharness-utils/cloudharness_utils/constants.py) should be extended/modified. + +### Generation of the base application skeleton + +The (re-)generation REST API is obtain through the [`harness-generate`](../tools/deployment-cli-tools/harness-generate) command. +The command parses the name of the application, gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. + +The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. +All of these functions are located in the `harness-generate` script. + +Under the hood, the `generate_servers(...)` function uses the `generate_fastapi_server(...)` and the `generate_server(...)` function that are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. +The generation of one type of servers over another one is bound to the existence of a `genapi.sh` file: + +```python +def generate_servers(root_path, interactive=False): + # ... + if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): + # fastapi server --> use the genapi.sh script + generate_fastapi_server(application_root) + else: + generate_server(application_root) +``` + +The `generate_clients(...)` function also uses `generate_python_client(...)` and `generate_ts_client(...)` from the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. +The `generate_ts_client(...)` function is called only if there is folder named `frontend` in the application directory structure: + +```python +def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): + # ... + app_dir = os.path.dirname(os.path.dirname(openapi_file)) + generate_python_client(app_name, openapi_file, + client_src_path, lib_name=client_lib_name) + if os.path.exists(os.path.join(app_dir, 'frontend')): + generate_ts_client(openapi_file) +``` + +### Generation of the application deployment files + +The generation of the deployment files is obtain through the [`harness-deployment`] script. +The script uses various arguments to configure properly the deployment of the application as well as some debug configuration helper for vscode as shown in the snippet below: + +```python +helm_values = create_helm_chart( # <1> + root_paths, + tag=args.tag, + registry=args.registry, + domain=args.domain, + local=args.local, + secured=not args.unsecured, + output_path=args.output_path, + exclude=args.exclude, + include=args.include, + registry_secret=args.registry_secret, + tls=not args.no_tls, + env=envs, + namespace=args.namespace +) + +merged_root_paths = preprocess_build_overrides( + root_paths=root_paths, helm_values=helm_values) # <2> + +if not args.no_cd_gen and envs: + create_codefresh_deployment_scripts( # <3> + merged_root_paths, + include=args.include, + exclude=args.exclude, + envs=envs, + base_image_name=helm_values['name'], + helm_values=helm_values) + +# ... + +create_skaffold_configuration(merged_root_paths, helm_values) # <4> + +#... + +hosts_info(helm_values) # <5> +``` + +First, the code for the Helm chart files is generated using the `create_helm_chart(...)` function (`<1>`). +Then, the dictionnary of values for the Helm configuration is preprocessed to change some path (`<2>`). +If necessary, the codefresh deployment scripts are generated (`<3>`) using the `create_codefresh_deployment_scripts(...)` function. +Then, the skaffold configuration is generated using the dictionnary generated for the Helm configuration and the `create_skaffold_configuration(...)` function (`<4>`). +Finally, the information about the host IP, domain names, ... is displayed on stdout using the `hosts_info(...)` function (`<5>`). + +#### Generation of the Helm chart + +The generation of the Helm chart relies on the `create_helm_chart(...)` function which is located in the [`helm.py`](../tools/deployment-cli-tools/ch_cli_tools/helm.py) module. + +This function creates an instance of `CloudHarnessHelm` and processes the values inserted in this instance. The `process_values(...)` method on the `CloudHarnessHelm` class creates the result dictionnary with all the required keys and finally returns them wrapped in an instance of [`HarnessMainConfig`](../libraries/models/cloudharness_model/models/harness_main_config.py). +This class extracts information from the dictionnary and gives quick access to them through specific getter/setters (this code, as well as the code located in [`cloudharness_model/models/`](../libraries/models/cloudharness_model/models/) is actually generated from the OpenAPI specification of the CloudHarness concepts located in [`cloudharness_model/api/openapi.yaml`](../libraries/models/api/openapi.yaml), which contains details about the different keys and concepts that can be used for a basic CloudHarness application configuration). +The intermediate dictionnary created for the Helm chart generation is complex and contains many sub-dictionnaries that all capture a part of the Helm chart. +The initializer receives as arguments information about the application, its location, the namespace of the application when it will run in Kubernete, ..., processes the information, creates the dictionnary and save the results as a YAML file using the `merge_to_yaml_file(...)` from the [`utils.py`](../tools/deployment-cli-tools/ch_cli_tools/utils.py) module to the path partially specified by constants from the `constants.py` module. + +The `helm.py` module also defines the `hosts_info(...)` function that displays information about the domain, subdomains, IP, ... of the application to be deployed. + +#### Generation of the Codefresh deployment files + +The generation of the Codefresh deployement files is entirely done from the [`codefresh.py`](../tools/deployment-cli-tools/ch_cli_tools/codefresh.py) module using the `create_codefresh_deployment_scripts(...)` function. +This function takes as parameter the Helm configuration generated by the `create_helm_chart(...)` function and generates different codefresh deployment script depending on environments (*e.g.:* dev, stage, prod). + + +#### Generation of the skaffold configuration + +The skaffold configuration is generated by the `create_skaffold_configuration(...)` function from the [`skaffold.py`](../tools/deployment-cli-tools/ch_cli_tools/skaffold.py) module. +This function also generates the skaffold entries for the Dockerfiles of the micro-services used in the application. +The skaffold generation is based on the [`skaffold-template.yaml`](../deployment-configuration/skaffold-template.yaml) from the CloudHarness project located in [`deployment-configuration`](../deployment-configuration/). +This base configuration is merged with the configuration dedicated to a specific project and which is located in the `deployment-configuration` folder of the project. +Finally, once all the requiered information are injected in the skaffold configuration dictionnary, the dictionnary is saved as a YAML file in the `deployment/skaffold.yaml` file in the project directory. + +#### How to extend the deployment generation + +**Add a new configuration deployment system**. If a new configuration system is targeted by the generation (not Helm chart or skaffold), a new kind of configuration should be added to CloudHarness, and the code should be rewritten to produce a configuration dictionnary from a new file that is not `Chart.yaml`, or that is compatible with this one. +This new kind of argument should be parsed from the command line in the `harness-configuration` script to take into account the new target. +A new module should be added where `helm.py` and `skaffold.py` are located. +This new module should be responsible for taking information from the application dictionnary and use this information to generate a new dictionnary in memory with the missing information that are necessary to properly build the deployment scripts. +Finally, new unit tests should be added to the [deployment-cli-tools/tests](../tools/deployment-cli-tools/tests/) folder. +If a new kind of class must be coded to get all the information of the configuration (like the `CloudHarnessHelm` class), then the [`cloudharness_model/api/openapi.yaml`](../libraries/models/api/openapi.yaml) must be modified to introduce the new type of object that will be manipulated to represent the documentation, and the OpenAPI model should be generated again. + + +## The CloudHarness runtime + +The CloudHarness runtime is located in the [`libraries`](../libraries/) folder. +The runtime library defines a set of concepts and functions to help the various micro-services to communicate together. +The code is organised as such: + +```bash +libraries/ +├── api # Contains the CloudHarness OpenAPI specification +│ ├── config.json # A configuration file to direct the OpenAPI code generation +│ └── openapi.yaml # The OpenAPI specification +├── client # A Python client to access the CloudHardness API +│ └── cloudharness_cli # The programmatic Python client API to access CloudHarness. This code is generated +├── cloudharness-common # The runtime library in itself that is used for dedicated tasks and applications +├── cloudharness-utils # Some shared utils between the CloudHarness CLI tools and the runtime (constants.py) +└── models # The CloudHarness model (generated from the OpenAPI specification) + ├── api # Copy of the artifacts that have been used for the generation (from libraries/api) + │ ├── config.json + │ └── openapi.yaml + └── cloudharness_model # The generated CloudHarness Python model +``` + +The `cloudharness-common` folder is where is located most of the custom code for the various tasks and applications. +The code is structured as this: + +```bash +cloudharness +├── applications.py # Contains helpers regarding about the application configuration +├── auth # Primitives related to authentication and KeyCloak +│ ├── exceptions.py # Dedicated exceptions +│ ├── __init__.py +│ ├── keycloak.py # Implementation specific code for KeyCloak, contains helpers to create KeyCloak clients, get tokens, configuration, ... +│ └── quota.py # Manage a quota by users +├── errors.py # Dedicated exceptions +├── events # Primitives related to event streaming with Kafka +│ ├── client.py # Functions related to the connexion to the Kafka broker +│ ├── decorators.py # Decorator implementation to easily send the result of a function to Kafka +│ ├── __init__.py +├── infrastructure # Primitives related to the management of the infrastructure with Kubernete +│ ├── __init__.py +│ ├── k8s.py # Functions for Kubernete namespace and pod managment +├── __init__.py +├── middleware # Manage user authentication header injection +│ ├── django.py # Way of injecting the auth token in requests for Django +│ ├── flask.py # Way of injecting the auth token in requests for Flask +│ ├── __init__.py +├── sentry # Primitives for sentry initialisation +│ └── __init__.py +├── service # Additional services to handle Persistent Volum Claim in Kubernetes +│ ├── __init__.py +│ ├── pvc.py +│ └── templates +│ └── pvc.yaml +├── utils # Set of helpers +│ ├── config.py # Helper class for the CloudHarness configuration +│ ├── env.py # Helper for the env variables in configurations +│ ├── __init__.py +│ ├── secrets.py # Helper class for the CloudHarness application secrets +│ ├── server.py # Helpers for flask/server bootstrapping +└── workflows # Primitives for the management of workflows + ├── argo.py # Helpers and function to access the Argo REST API + ├── __init__.py + ├── operations.py # Functions to create new Argo operations + ├── tasks.py # Functions to create new Argo tasks + └── utils.py # Helpers to get information from the pods that executes operations and tasks +``` \ No newline at end of file diff --git a/docs/model/ApplicationDependenciesConfig.md b/docs/model/ApplicationDependenciesConfig.md index 8d3ba917..a59b2cec 100644 --- a/docs/model/ApplicationDependenciesConfig.md +++ b/docs/model/ApplicationDependenciesConfig.md @@ -11,6 +11,7 @@ Key | Input Type | Accessed Type | Description | Notes **[hard](#hard)** | list, tuple, | tuple, | Hard dependencies indicate that the application may not start without these other applications. | [optional] **[soft](#soft)** | list, tuple, | tuple, | Soft dependencies indicate that the application will work partially without these other applications. | [optional] **[build](#build)** | list, tuple, | tuple, | Hard dependencies indicate that the application Docker image build requires these base/common images | [optional] +**[git](#git)** | list, tuple, | tuple, | | [optional] **any_string_name** | dict, frozendict.frozendict, str, date, datetime, int, float, bool, decimal.Decimal, None, list, tuple, bytes, io.FileIO, io.BufferedReader | frozendict.frozendict, str, BoolClass, decimal.Decimal, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional] # hard @@ -55,5 +56,17 @@ Class Name | Input Type | Accessed Type | Description | Notes ------------- | ------------- | ------------- | ------------- | ------------- items | str, | str, | | +# git + +## Model Type Info +Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- +list, tuple, | tuple, | | + +### Tuple Items +Class Name | Input Type | Accessed Type | Description | Notes +------------- | ------------- | ------------- | ------------- | ------------- +[**GitDependencyConfig**](GitDependencyConfig.md) | [**GitDependencyConfig**](GitDependencyConfig.md) | [**GitDependencyConfig**](GitDependencyConfig.md) | | + [[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md) diff --git a/docs/model/GitDependencyConfig.md b/docs/model/GitDependencyConfig.md new file mode 100644 index 00000000..5faef206 --- /dev/null +++ b/docs/model/GitDependencyConfig.md @@ -0,0 +1,19 @@ +# cloudharness_model.model.git_dependency_config.GitDependencyConfig + +Defines a git repo to be cloned inside the application path + +## Model Type Info +Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- +dict, frozendict.frozendict, | frozendict.frozendict, | Defines a git repo to be cloned inside the application path | + +### Dictionary Keys +Key | Input Type | Accessed Type | Description | Notes +------------ | ------------- | ------------- | ------------- | ------------- +**branch_tag** | str, | str, | | +**url** | str, | str, | | +**path** | str, | str, | Defines the path where the repo is cloned. default: /git | [optional] +**any_string_name** | dict, frozendict.frozendict, str, date, datetime, int, float, bool, decimal.Decimal, None, list, tuple, bytes, io.FileIO, io.BufferedReader | frozendict.frozendict, str, BoolClass, decimal.Decimal, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional] + +[[Back to Model list]](../../README.md#documentation-for-models) [[Back to API list]](../../README.md#documentation-for-api-endpoints) [[Back to README]](../../README.md) + diff --git a/docs/model/HarnessMainConfig.md b/docs/model/HarnessMainConfig.md index 9f00bc2b..fc0f29da 100644 --- a/docs/model/HarnessMainConfig.md +++ b/docs/model/HarnessMainConfig.md @@ -21,6 +21,7 @@ Key | Input Type | Accessed Type | Description | Notes **backup** | [**BackupConfig**](BackupConfig.md) | [**BackupConfig**](BackupConfig.md) | | [optional] **name** | str, | str, | Base name | [optional] **task-images** | [**SimpleMap**](SimpleMap.md) | [**SimpleMap**](SimpleMap.md) | | [optional] +**build_hash** | str, | str, | | [optional] **any_string_name** | dict, frozendict.frozendict, str, date, datetime, uuid.UUID, int, float, decimal.Decimal, bool, None, list, tuple, bytes, io.FileIO, io.BufferedReader, | frozendict.frozendict, str, decimal.Decimal, BoolClass, NoneClass, tuple, bytes, FileIO | any string name can be used but the value must be the correct type | [optional] # env diff --git a/libraries/cloudharness-utils/cloudharness_utils/constants.py b/libraries/cloudharness-utils/cloudharness_utils/constants.py index c4fb4f6c..c26ed542 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/constants.py +++ b/libraries/cloudharness-utils/cloudharness_utils/constants.py @@ -43,7 +43,7 @@ CD_E2E_TEST_STEP = 'tests_e2e' CD_STEP_PUBLISH = 'publish' BUILD_FILENAMES = ('node_modules',) - +CD_BUILD_STEP_DEPENDENCIES = 'post_main_clone' E2E_TESTS_DIRNAME = 'e2e' API_TESTS_DIRNAME = 'api' diff --git a/libraries/models/api/openapi.yaml b/libraries/models/api/openapi.yaml index 577bff53..8b961031 100644 --- a/libraries/models/api/openapi.yaml +++ b/libraries/models/api/openapi.yaml @@ -68,6 +68,11 @@ components: type: array items: type: string + git: + description: '' + type: array + items: + $ref: '#/components/schemas/GitDependencyConfig' DeploymentResourcesConf: description: '' type: object @@ -834,6 +839,9 @@ components: task-images: $ref: '#/components/schemas/SimpleMap' description: '' + build_hash: + description: '' + type: string additionalProperties: true SimpleMap: description: '' @@ -846,3 +854,22 @@ components: example: quota-ws-max: 5 quota-storage-max: 1G + GitDependencyConfig: + title: Root Type for GitDependencyConfig + description: Defines a git repo to be cloned inside the application path + required: + - branch_tag + - url + type: object + properties: + url: + type: string + branch_tag: + type: string + path: + description: 'Defines the path where the repo is cloned. default: /git' + type: string + example: + url: 'https://github.com/MetaCell/nwb-explorer.git' + branch_tag: master + path: /git diff --git a/libraries/models/cloudharness_model/models/__init__.py b/libraries/models/cloudharness_model/models/__init__.py index 78d8f9fc..dbfb1175 100644 --- a/libraries/models/cloudharness_model/models/__init__.py +++ b/libraries/models/cloudharness_model/models/__init__.py @@ -25,6 +25,7 @@ from cloudharness_model.models.deployment_volume_spec_all_of import DeploymentVolumeSpecAllOf from cloudharness_model.models.e2_e_tests_config import E2ETestsConfig from cloudharness_model.models.file_resources_config import FileResourcesConfig +from cloudharness_model.models.git_dependency_config import GitDependencyConfig from cloudharness_model.models.harness_main_config import HarnessMainConfig from cloudharness_model.models.ingress_config import IngressConfig from cloudharness_model.models.ingress_config_all_of import IngressConfigAllOf diff --git a/libraries/models/cloudharness_model/models/application_dependencies_config.py b/libraries/models/cloudharness_model/models/application_dependencies_config.py index d4330359..7f4df0ba 100644 --- a/libraries/models/cloudharness_model/models/application_dependencies_config.py +++ b/libraries/models/cloudharness_model/models/application_dependencies_config.py @@ -6,8 +6,10 @@ from typing import List, Dict # noqa: F401 from cloudharness_model.models.base_model_ import Model +from cloudharness_model.models.git_dependency_config import GitDependencyConfig from cloudharness_model import util +from cloudharness_model.models.git_dependency_config import GitDependencyConfig # noqa: E501 class ApplicationDependenciesConfig(Model): """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). @@ -15,7 +17,7 @@ class ApplicationDependenciesConfig(Model): Do not edit the class manually. """ - def __init__(self, hard=None, soft=None, build=None): # noqa: E501 + def __init__(self, hard=None, soft=None, build=None, git=None): # noqa: E501 """ApplicationDependenciesConfig - a model defined in OpenAPI :param hard: The hard of this ApplicationDependenciesConfig. # noqa: E501 @@ -24,22 +26,27 @@ def __init__(self, hard=None, soft=None, build=None): # noqa: E501 :type soft: List[str] :param build: The build of this ApplicationDependenciesConfig. # noqa: E501 :type build: List[str] + :param git: The git of this ApplicationDependenciesConfig. # noqa: E501 + :type git: List[GitDependencyConfig] """ self.openapi_types = { 'hard': List[str], 'soft': List[str], - 'build': List[str] + 'build': List[str], + 'git': List[GitDependencyConfig] } self.attribute_map = { 'hard': 'hard', 'soft': 'soft', - 'build': 'build' + 'build': 'build', + 'git': 'git' } self._hard = hard self._soft = soft self._build = build + self._git = git @classmethod def from_dict(cls, dikt) -> 'ApplicationDependenciesConfig': @@ -120,3 +127,26 @@ def build(self, build): """ self._build = build + + @property + def git(self): + """Gets the git of this ApplicationDependenciesConfig. + + # noqa: E501 + + :return: The git of this ApplicationDependenciesConfig. + :rtype: List[GitDependencyConfig] + """ + return self._git + + @git.setter + def git(self, git): + """Sets the git of this ApplicationDependenciesConfig. + + # noqa: E501 + + :param git: The git of this ApplicationDependenciesConfig. + :type git: List[GitDependencyConfig] + """ + + self._git = git diff --git a/libraries/models/cloudharness_model/models/git_dependency_config.py b/libraries/models/cloudharness_model/models/git_dependency_config.py new file mode 100644 index 00000000..b2a17690 --- /dev/null +++ b/libraries/models/cloudharness_model/models/git_dependency_config.py @@ -0,0 +1,122 @@ +# coding: utf-8 + +from __future__ import absolute_import +from datetime import date, datetime # noqa: F401 + +from typing import List, Dict # noqa: F401 + +from cloudharness_model.models.base_model_ import Model +from cloudharness_model import util + + +class GitDependencyConfig(Model): + """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + + Do not edit the class manually. + """ + + def __init__(self, url=None, branch_tag=None, path=None): # noqa: E501 + """GitDependencyConfig - a model defined in OpenAPI + + :param url: The url of this GitDependencyConfig. # noqa: E501 + :type url: str + :param branch_tag: The branch_tag of this GitDependencyConfig. # noqa: E501 + :type branch_tag: str + :param path: The path of this GitDependencyConfig. # noqa: E501 + :type path: str + """ + self.openapi_types = { + 'url': str, + 'branch_tag': str, + 'path': str + } + + self.attribute_map = { + 'url': 'url', + 'branch_tag': 'branch_tag', + 'path': 'path' + } + + self._url = url + self._branch_tag = branch_tag + self._path = path + + @classmethod + def from_dict(cls, dikt) -> 'GitDependencyConfig': + """Returns the dict as a model + + :param dikt: A dict. + :type: dict + :return: The GitDependencyConfig of this GitDependencyConfig. # noqa: E501 + :rtype: GitDependencyConfig + """ + return util.deserialize_model(dikt, cls) + + @property + def url(self): + """Gets the url of this GitDependencyConfig. + + + :return: The url of this GitDependencyConfig. + :rtype: str + """ + return self._url + + @url.setter + def url(self, url): + """Sets the url of this GitDependencyConfig. + + + :param url: The url of this GitDependencyConfig. + :type url: str + """ + if url is None: + raise ValueError("Invalid value for `url`, must not be `None`") # noqa: E501 + + self._url = url + + @property + def branch_tag(self): + """Gets the branch_tag of this GitDependencyConfig. + + + :return: The branch_tag of this GitDependencyConfig. + :rtype: str + """ + return self._branch_tag + + @branch_tag.setter + def branch_tag(self, branch_tag): + """Sets the branch_tag of this GitDependencyConfig. + + + :param branch_tag: The branch_tag of this GitDependencyConfig. + :type branch_tag: str + """ + if branch_tag is None: + raise ValueError("Invalid value for `branch_tag`, must not be `None`") # noqa: E501 + + self._branch_tag = branch_tag + + @property + def path(self): + """Gets the path of this GitDependencyConfig. + + Defines the path where the repo is cloned. default: /git # noqa: E501 + + :return: The path of this GitDependencyConfig. + :rtype: str + """ + return self._path + + @path.setter + def path(self, path): + """Sets the path of this GitDependencyConfig. + + Defines the path where the repo is cloned. default: /git # noqa: E501 + + :param path: The path of this GitDependencyConfig. + :type path: str + """ + + self._path = path diff --git a/libraries/models/cloudharness_model/models/harness_main_config.py b/libraries/models/cloudharness_model/models/harness_main_config.py index c75db6d3..7f18e82d 100644 --- a/libraries/models/cloudharness_model/models/harness_main_config.py +++ b/libraries/models/cloudharness_model/models/harness_main_config.py @@ -23,7 +23,7 @@ class HarnessMainConfig(Model): Do not edit the class manually. """ - def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace=None, mainapp=None, registry=None, tag=None, apps=None, env=None, privenv=None, backup=None, name=None, task_images=None): # noqa: E501 + def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace=None, mainapp=None, registry=None, tag=None, apps=None, env=None, privenv=None, backup=None, name=None, task_images=None, build_hash=None): # noqa: E501 """HarnessMainConfig - a model defined in OpenAPI :param local: The local of this HarnessMainConfig. # noqa: E501 @@ -52,6 +52,8 @@ def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace= :type name: str :param task_images: The task_images of this HarnessMainConfig. # noqa: E501 :type task_images: Dict[str, object] + :param build_hash: The build_hash of this HarnessMainConfig. # noqa: E501 + :type build_hash: str """ self.openapi_types = { 'local': bool, @@ -66,7 +68,8 @@ def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace= 'privenv': NameValue, 'backup': BackupConfig, 'name': str, - 'task_images': Dict[str, object] + 'task_images': Dict[str, object], + 'build_hash': str } self.attribute_map = { @@ -82,7 +85,8 @@ def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace= 'privenv': 'privenv', 'backup': 'backup', 'name': 'name', - 'task_images': 'task-images' + 'task_images': 'task-images', + 'build_hash': 'build_hash' } self._local = local @@ -98,6 +102,7 @@ def __init__(self, local=None, secured_gatekeepers=None, domain=None, namespace= self._backup = backup self._name = name self._task_images = task_images + self._build_hash = build_hash @classmethod def from_dict(cls, dikt) -> 'HarnessMainConfig': @@ -414,3 +419,26 @@ def task_images(self, task_images): """ self._task_images = task_images + + @property + def build_hash(self): + """Gets the build_hash of this HarnessMainConfig. + + # noqa: E501 + + :return: The build_hash of this HarnessMainConfig. + :rtype: str + """ + return self._build_hash + + @build_hash.setter + def build_hash(self, build_hash): + """Sets the build_hash of this HarnessMainConfig. + + # noqa: E501 + + :param build_hash: The build_hash of this HarnessMainConfig. + :type build_hash: str + """ + + self._build_hash = build_hash diff --git a/tools/clone.sh b/tools/clone.sh new file mode 100644 index 00000000..246c4d89 --- /dev/null +++ b/tools/clone.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +REPOSRC=$2 +LOCALREPO=$3 +BRANCH=$1 + +# We do it this way so that we can abstract if from just git later on +LOCALREPO_VC_DIR=$LOCALREPO/.git + +if [ ! -d $LOCALREPO_VC_DIR ] +then + git clone --branch $BRANCH $REPOSRC $LOCALREPO +else + cd $LOCALREPO + git pull origin $BRANCH +fi + +# End \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py index 6432233b..d6b699a5 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/codefresh.py +++ b/tools/deployment-cli-tools/ch_cli_tools/codefresh.py @@ -1,5 +1,6 @@ import os from os.path import join, relpath, exists, dirname +from cloudharness_model.models.git_dependency_config import GitDependencyConfig import requests import logging from cloudharness_model.models.api_tests_config import ApiTestsConfig @@ -33,6 +34,27 @@ def literal_presenter(dumper, data): yaml.add_representer(str, literal_presenter) +def get_main_domain(url): + try: + url = url.split("//")[1].split("/")[0] + if "gitlab" in url: + return "gitlab" + if "bitbucket" in url: + return "bitbucket" + return "github" + except: + return "${{ DEFAULT_REPO }}" + +def clone_step_spec(conf: GitDependencyConfig, context_path: str): + return { + "title": f"Cloning {os.path.basename(conf.url)} repository...", + "type": "git-clone", + "repo": conf.url, + "revision": conf.branch_tag, + "working_directory": join(context_path, "dependencies", conf.path or os.path.basename(conf.url)), + "git": get_main_domain(conf.url) # Cannot really tell what's the git config name, usually the name of the repo + } + def write_env_file(helm_values: HarnessMainConfig, filename, registry_secret=None): env = {} logging.info("Create env file with image info %s", filename) @@ -152,6 +174,10 @@ def codefresh_steps_from_base_path(base_path, build_step, fixed_context=None, in # Skip excluded apps continue + if app_config and app_config.dependencies and app_config.dependencies.git: + for dep in app_config.dependencies.git: + steps[CD_BUILD_STEP_DEPENDENCIES]['steps'].append(clone_step_spec(dep, base_path)) + build = None if build_step in steps: build = codefresh_app_build_spec( diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index 4c75a909..dc4a6cba 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -15,7 +15,7 @@ from . import HERE, CH_ROOT from cloudharness_utils.constants import TEST_IMAGES_PATH, VALUES_MANUAL_PATH, HELM_CHART_PATH, APPS_PATH, HELM_PATH, \ DEPLOYMENT_CONFIGURATION_PATH, BASE_IMAGES_PATH, STATIC_IMAGES_PATH -from .utils import get_cluster_ip, get_image_name, env_variable, get_sub_paths, guess_build_dependencies_from_dockerfile, image_name_from_dockerfile_path, \ +from .utils import get_cluster_ip, get_git_commit_hash, get_image_name, env_variable, get_sub_paths, guess_build_dependencies_from_dockerfile, image_name_from_dockerfile_path, \ get_template, merge_configuration_directories, merge_to_yaml_file, dict_merge, app_name_from_path, \ find_dockerfiles_paths @@ -323,6 +323,8 @@ def __finish_helm_values(self, values): """ if self.registry: logging.info(f"Registry set: {self.registry}") + + if self.local: values['registry']['secret'] = '' if self.registry_secret: @@ -330,6 +332,7 @@ def __finish_helm_values(self, values): values['registry']['name'] = self.registry values['registry']['secret'] = self.registry_secret values['tag'] = self.tag + values['build_hash'] = get_git_commit_hash(self.root_paths[-1]) # Fix: Call the defined function to get the git commit hash if self.namespace: values['namespace'] = self.namespace values['secured_gatekeepers'] = self.secured diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index c0de5764..bd2f4da4 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -4,7 +4,7 @@ import time from os.path import join, relpath, basename, exists, abspath -from cloudharness_model import ApplicationTestConfig, HarnessMainConfig +from cloudharness_model import ApplicationTestConfig, HarnessMainConfig, GitDependencyConfig from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ BASE_IMAGES_PATH, STATIC_IMAGES_PATH @@ -57,7 +57,8 @@ def build_artifact(image_name, context_path, requirements=None, dockerfile_path= def process_build_dockerfile(dockerfile_path, root_path, global_context=False, requirements=None, app_name=None): if app_name is None: app_name = app_name_from_path(basename(dockerfile_path)) - if app_name in helm_values[KEY_TASK_IMAGES] or app_name.replace("-", "_") in helm_values.apps: + app_key = app_name.replace("-", "_") + if app_name in helm_values[KEY_TASK_IMAGES] or app_key in helm_values.apps: context_path = relpath_if(root_path, output_path) if global_context else relpath_if(dockerfile_path, output_path) builds[app_name] = context_path @@ -68,6 +69,10 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r dockerfile_path=relpath(dockerfile_path, output_path), requirements=requirements or guess_build_dependencies_from_dockerfile(dockerfile_path) ) + if app_key in helm_values.apps and helm_values.apps[app_key].harness.dependencies and helm_values.apps[app_key].harness.dependencies.git: + artifacts[app_name]['hooks'] = { + 'before': [git_clone_hook(conf, context_path) for conf in helm_values.apps[app_key].harness.dependencies.git] + } for root_path in root_paths: skaffold_conf = dict_merge(skaffold_conf, get_template( @@ -189,6 +194,16 @@ def identify_unicorn_based_main(candidates): output_path, 'skaffold.yaml')) return skaffold_conf +def git_clone_hook(conf: GitDependencyConfig, context_path: str): + return { + 'command': [ + 'sh', + 'tools/clone.sh', + conf.branch_tag, + conf.url, + join(context_path, "dependencies", conf.path or os.path.basename(conf.url)) + ] + } def create_vscode_debug_configuration(root_paths, helm_values): logging.info( diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 8b7e20be..375b35ec 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -59,7 +59,22 @@ def find_subdirs(base_path): def find_dockerfiles_paths(base_directory): - return find_file_paths(base_directory, 'Dockerfile') + all_dockerfiles = find_file_paths(base_directory, 'Dockerfile') + + # We want to remove all dockerfiles that are not in a git repository + # This will exclude the cloned dependencies and other repos cloned for convenience + dockerfiles_without_git = [] + + for dockerfile in all_dockerfiles: + directory = dockerfile + while not os.path.samefile(directory, base_directory): + if os.path.exists(os.path.join(directory, '.git')): + break + directory = os.path.dirname(directory) + else: + dockerfiles_without_git.append(dockerfile.replace(os.sep, "/")) + + return tuple(dockerfiles_without_git) def get_parent_app_name(app_relative_path): @@ -410,3 +425,14 @@ def search_word_in_folder(folder, word): else: raise Exception(f'Migration Error: {folder} grep failed with return code {p.returncode} and output {output}') return list(filter(filter_empty_strings, matches)) + + +def get_git_commit_hash(path): + # return the short git commit hash in that path + # if the path is not a git repo, return None + + try: + return subprocess.check_output( + ['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode("utf-8").strip() + except: + return None diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/dependencies/a/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/myapp/dependencies/a/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml index 434b042b..e386702a 100644 --- a/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml +++ b/tools/deployment-cli-tools/tests/resources/applications/myapp/deploy/values.yaml @@ -6,6 +6,12 @@ harness: - legacy build: - cloudharness-flask + git: + - url: https://github.com/a/b.git + branch_tag: master + - url: https://github.com/c/d.git + branch_tag: v1.0.0 + path: myrepo test: unit: commands: diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index aee93d37..7e3abd4d 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -9,6 +9,9 @@ CLOUDHARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) BUILD_MERGE_DIR = "./build/test_deployment" +myapp_path = os.path.join(HERE, "resources/applications/myapp") +if not os.path.exists(os.path.join(myapp_path, "dependencies/a/.git")): + os.makedirs(os.path.join(myapp_path, "dependencies/a/.git")) def test_create_codefresh_configuration(): values = create_helm_chart( @@ -123,6 +126,8 @@ def test_create_codefresh_configuration(): assert len( tstep['commands']) == 2, "Unit test commands are not properly loaded from the unit test configuration file" assert tstep['commands'][0] == "tox", "Unit test commands are not properly loaded from the unit test configuration file" + + assert len(l1_steps[CD_BUILD_STEP_DEPENDENCIES]['steps']) == 3, "3 clone steps should be included as we have 2 dependencies from myapp, plus cloudharness" finally: shutil.rmtree(BUILD_MERGE_DIR) diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index 94998892..04cf4420 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -86,7 +86,8 @@ def test_create_skaffold_configuration(): a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/myapp') assert os.path.samefile(myapp_artifact['context'], join( RESOURCES, 'applications/myapp')) - + assert myapp_artifact['hooks']['before'], 'The hook for dependencies should be included' + assert len(myapp_artifact['hooks']['before']) == 2, 'The hook for dependencies should include 2 clone commands' accounts_artifact = next( a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/accounts') assert os.path.samefile(accounts_artifact['context'], '/tmp/build/applications/accounts') diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index abae7137..d1ec8d6a 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -72,3 +72,15 @@ def test_search_word_in_file(): def test_search_word_in_folder(): assert len(search_word_in_folder(os.path.join(HERE, './resources/applications/migration_app/'), "CLOUDHARNESS_BASE_DEBIAN")) == 2 + + +def test_find_dockerfile_paths(): + + myapp_path = os.path.join(HERE, "resources/applications/myapp") + if not os.path.exists(os.path.join(myapp_path, "dependencies/a/.git")): + os.makedirs(os.path.join(myapp_path, "dependencies/a/.git")) + + dockerfiles = find_dockerfiles_paths(myapp_path) + assert len(dockerfiles) == 2 + assert next(d for d in dockerfiles if d.endswith("myapp")), "Must find the Dockerfile in the root directory" + assert next(d for d in dockerfiles if d.endswith("myapp/tasks/mytask")), "Must find the Dockerfile in the tasks directory"