diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index 1e6742a5..774234b9 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -3,7 +3,7 @@ import json import time -from os.path import join, dirname, relpath, basename +from os.path import join, relpath, basename, exists, abspath from cloudharness_model import ApplicationTestConfig, HarnessMainConfig from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ @@ -51,7 +51,7 @@ def build_artifact(image_name, context_path, requirements=None, dockerfile_path= in requirements] return artifact_spec - + base_images = set() def process_build_dockerfile(dockerfile_path, root_path, global_context=False, requirements=None, app_name=None): @@ -78,13 +78,13 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r for dockerfile_path in base_dockerfiles: process_build_dockerfile(dockerfile_path, root_path, global_context=True) - - + + release_config = skaffold_conf['deploy']['helm']['releases'][0] release_config['name'] = helm_values.namespace release_config['namespace'] = helm_values.namespace release_config['artifactOverrides'][KEY_APPS] = {} - + static_images = set() for root_path in root_paths: static_dockerfiles = find_dockerfiles_paths( @@ -92,7 +92,7 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r for dockerfile_path in static_dockerfiles: process_build_dockerfile(dockerfile_path, root_path) - + for root_path in root_paths: apps_path = join(root_path, APPS_PATH) @@ -138,22 +138,45 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r } } - flask_main = find_file_paths(context_path, '__main__.py') - - if flask_main: + mains_candidates = find_file_paths(context_path, '__main__.py') + + def identify_unicorn_based_main(candidates): + import re + gunicorn_pattern = re.compile(r"gunicorn") + for candidate in candidates: + dockerfile_path = f"{candidate}/.." + while not exists(f"{dockerfile_path}/Dockerfile") and abspath(dockerfile_path) != abspath(root_path): + dockerfile_path += "/.." + dockerfile = f"{dockerfile_path}/Dockerfile" + if not exists(dockerfile): + continue + with open(dockerfile, 'r') as file: + if re.search(gunicorn_pattern, file.read()): + return candidate + requirements = f"{candidate}/../requirements.txt" + if not exists(requirements): + continue + with open(requirements, 'r') as file: + if re.search(gunicorn_pattern, file.read()): + return candidate + return None + + task_main_file = identify_unicorn_based_main(mains_candidates) + + if task_main_file: release_config['overrides']['apps'][app_key] = \ { 'harness': { 'deployment': { 'command': ['python'], - 'args': [f'/usr/src/app/{os.path.basename(flask_main[0])}/__main__.py'] + 'args': [f'/usr/src/app/{os.path.basename(task_main_file)}/__main__.py'] } } } - + test_config: ApplicationTestConfig = helm_values.apps[app_key].harness.test if test_config.unit.enabled and test_config.unit.commands: - + skaffold_conf['test'].append(dict( image=get_image_tag(app_name), custom=[dict(command="docker run $IMAGE " + cmd) for cmd in test_config.unit.commands] @@ -209,7 +232,7 @@ def get_image_tag(name): "/usr/src/app"), } }) - + if not os.path.exists(os.path.dirname(vscode_launch_path)): os.makedirs(os.path.dirname(vscode_launch_path)) diff --git a/tools/deployment-cli-tools/tests/resources_buggy/.dockerignore b/tools/deployment-cli-tools/tests/resources_buggy/.dockerignore new file mode 100644 index 00000000..bba89d5b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/.dockerignore @@ -0,0 +1,22 @@ +**/node_modules +.tox +docs +applications +/infrastructure +/blueprint +test +/tools/deployment-cli-tools +.github +.git +.vscode +/deployment +skaffold.yaml +*.egg-info +__pycache__ +.hypothesis +.coverage +.pytest_cache +/application-templates +/deployment-configuration +/cloud-harness +.openapi-generator \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values-prod.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values-prod.yaml new file mode 100644 index 00000000..d4d54b1f --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values-prod.yaml @@ -0,0 +1,7 @@ +kafka: + resources: + limits: + cpu: overridden-prod + requests: + cpu: 50m + memory: 100Mi diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values.yaml new file mode 100644 index 00000000..300997d5 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values.yaml @@ -0,0 +1,8 @@ +kafka: + resources: + limits: + cpu: 600m + memory: overridden + requests: + cpu: 50m + memory: 100Mi diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/.dockerignore b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/.dockerignore new file mode 100644 index 00000000..669be813 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/.dockerignore @@ -0,0 +1 @@ +*.ignored \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile new file mode 100644 index 00000000..533a9c95 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_FLASK +FROM $CLOUDHARNESS_FLASK + +RUN pip install gunicorn \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/resources/aresource.txt b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/resources/aresource.txt new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/templates/mytemplate.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/templates/mytemplate.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-dev.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-dev.yaml new file mode 100644 index 00000000..0b7c9a77 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-dev.yaml @@ -0,0 +1,11 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + - my-common +a: b +dev: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-test.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-test.yaml new file mode 100644 index 00000000..85ecbfae --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-test.yaml @@ -0,0 +1,2 @@ +a: test +test: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withmongo.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withmongo.yaml new file mode 100644 index 00000000..f3519b50 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withmongo.yaml @@ -0,0 +1,4 @@ +harness: + database: + auto: true + type: mongo \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withoutdb.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withoutdb.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withpostgres.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withpostgres.yaml new file mode 100644 index 00000000..4b260dbf --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withpostgres.yaml @@ -0,0 +1,2 @@ +harness: + database: {auto: true, type: postgres} \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values.yaml new file mode 100644 index 00000000..434b042b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values.yaml @@ -0,0 +1,14 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + test: + unit: + commands: + - tox + - echo "hello" +a: b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/myapp_code/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/myapp_code/__main__.py new file mode 100644 index 00000000..3118318a --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/myapp_code/__main__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +from cloudharness.utils.server import init_flask, main + + +app = init_flask(title="Cloudharness sample application", webapp=True) + +if __name__ == '__main__': + main() diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/tasks/mytask/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/tasks/mytask/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/venv/matplotlib/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/venv/matplotlib/__main__.py new file mode 100644 index 00000000..37de2b0b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/venv/matplotlib/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from something import something_else + + +def fake_content(): + ... \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore new file mode 100644 index 00000000..669be813 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore @@ -0,0 +1 @@ +*.ignored \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile new file mode 100644 index 00000000..de89f0ed --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_FLASK +FROM $CLOUDHARNESS_FLASK + +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/resources/aresource.txt b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/resources/aresource.txt new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/templates/mytemplate.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/templates/mytemplate.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml new file mode 100644 index 00000000..0b7c9a77 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml @@ -0,0 +1,11 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + - my-common +a: b +dev: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml new file mode 100644 index 00000000..85ecbfae --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml @@ -0,0 +1,2 @@ +a: test +test: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml new file mode 100644 index 00000000..f3519b50 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml @@ -0,0 +1,4 @@ +harness: + database: + auto: true + type: mongo \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withoutdb.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withoutdb.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml new file mode 100644 index 00000000..4b260dbf --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml @@ -0,0 +1,2 @@ +harness: + database: {auto: true, type: postgres} \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml new file mode 100644 index 00000000..434b042b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml @@ -0,0 +1,14 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + test: + unit: + commands: + - tox + - echo "hello" +a: b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py new file mode 100644 index 00000000..3118318a --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +from cloudharness.utils.server import init_flask, main + + +app = init_flask(title="Cloudharness sample application", webapp=True) + +if __name__ == '__main__': + main() diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt new file mode 100644 index 00000000..283e1305 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt @@ -0,0 +1 @@ +gunicorn \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/tasks/mytask/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/tasks/mytask/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py new file mode 100644 index 00000000..37de2b0b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from something import something_else + + +def fake_content(): + ... \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/infrastructure/base-images/cloudharness-base/testfile b/tools/deployment-cli-tools/tests/resources_buggy/infrastructure/base-images/cloudharness-base/testfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/infrastructure/common-images/my-common/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/infrastructure/common-images/my-common/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index 98036b21..94998892 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -7,6 +7,7 @@ HERE = os.path.dirname(os.path.realpath(__file__)) RESOURCES = os.path.join(HERE, 'resources') +RESOURCES_BUGGY = os.path.join(HERE, 'resources_buggy') OUT = '/tmp/deployment' CLOUDHARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) @@ -70,7 +71,7 @@ def test_create_skaffold_configuration(): a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/cloudharness-flask') - assert os.path.samefile(cloudharness_flask_artifact['context'], + assert os.path.samefile(cloudharness_flask_artifact['context'], join(CLOUDHARNESS_ROOT, 'infrastructure/common-images/cloudharness-flask') ) @@ -102,3 +103,75 @@ def test_create_skaffold_configuration(): shutil.rmtree(OUT) shutil.rmtree(BUILD_DIR) + + +def test_create_skaffold_configuration_with_conflicting_dependencies(tmp_path): + values = create_helm_chart( + [CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + output_path=OUT, + include=['myapp'], + exclude=['events'], + domain="my.local", + namespace='test', + env='dev', + local=False, + tag=1, + registry='reg' + ) + root_paths = preprocess_build_overrides( + root_paths=[CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + helm_values=values, + merge_build_path=str(tmp_path) + ) + + sk = create_skaffold_configuration( + root_paths=root_paths, + helm_values=values, + output_path=OUT + ) + + releases = sk['deploy']['helm']['releases'] + assert len(releases) == 1 # Ensure we only found 1 deployment (for myapp) + + release = releases[0] + assert 'myapp' in release['overrides']['apps'] + assert 'matplotlib' not in release['overrides']['apps'] + + myapp_config = release['overrides']['apps']['myapp'] + assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' + + +def test_create_skaffold_configuration_with_conflicting_dependencies_requirements_file(tmp_path): + values = create_helm_chart( + [CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + output_path=OUT, + include=['myapp2'], + exclude=['events'], + domain="my.local", + namespace='test', + env='dev', + local=False, + tag=1, + registry='reg' + ) + root_paths = preprocess_build_overrides( + root_paths=[CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + helm_values=values, + merge_build_path=str(tmp_path) + ) + + sk = create_skaffold_configuration( + root_paths=root_paths, + helm_values=values, + output_path=OUT + ) + + releases = sk['deploy']['helm']['releases'] + assert len(releases) == 1 # Ensure we only found 1 deployment (for myapp) + + release = releases[0] + assert 'myapp2' in release['overrides']['apps'] + assert 'matplotlib' not in release['overrides']['apps'] + + myapp_config = release['overrides']['apps']['myapp2'] + assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' \ No newline at end of file