From 717bb12bf8f131f2a404118336a08438f0d13b27 Mon Sep 17 00:00:00 2001 From: aranega Date: Tue, 17 Oct 2023 00:39:00 -0600 Subject: [PATCH] CH-87 Add detection of gunicorn in Dockerfile or requirements.txt --- .../ch_cli_tools/skaffold.py | 28 ++++++++++----- .../applications/myapp/Dockerfile | 4 ++- .../applications/myapp2/.dockerignore | 1 + .../applications/myapp2/Dockerfile | 4 +++ .../myapp2/deploy/resources/aresource.txt | 0 .../myapp2/deploy/templates/mytemplate.yaml | 0 .../myapp2/deploy/values-dev.yaml | 11 ++++++ .../myapp2/deploy/values-test.yaml | 2 ++ .../myapp2/deploy/values-withmongo.yaml | 4 +++ .../myapp2/deploy/values-withoutdb.yaml | 0 .../myapp2/deploy/values-withpostgres.yaml | 2 ++ .../applications/myapp2/deploy/values.yaml | 14 ++++++++ .../myapp2/myapp_code/__main__.py | 9 +++++ .../applications/myapp2/requirements.txt | 1 + .../myapp2/tasks/mytask/Dockerfile | 0 .../myapp2/venv/matplotlib/__main__.py | 7 ++++ .../tests/test_skaffold.py | 36 +++++++++++++++++++ 17 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/resources/aresource.txt create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/templates/mytemplate.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withoutdb.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/tasks/mytask/Dockerfile create mode 100644 tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py index 8aba2633..774234b9 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/skaffold.py +++ b/tools/deployment-cli-tools/ch_cli_tools/skaffold.py @@ -3,7 +3,7 @@ import json import time -from os.path import join, dirname, relpath, basename +from os.path import join, relpath, basename, exists, abspath from cloudharness_model import ApplicationTestConfig, HarnessMainConfig from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \ @@ -140,24 +140,36 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r mains_candidates = find_file_paths(context_path, '__main__.py') - def identify_flask_main(candidates): + def identify_unicorn_based_main(candidates): import re - init_flask_pattern = re.compile(r"init_flask\(") + gunicorn_pattern = re.compile(r"gunicorn") for candidate in candidates: - with open(f"{candidate}/__main__.py", 'r') as file: - if re.search(init_flask_pattern, file.read()): + dockerfile_path = f"{candidate}/.." + while not exists(f"{dockerfile_path}/Dockerfile") and abspath(dockerfile_path) != abspath(root_path): + dockerfile_path += "/.." + dockerfile = f"{dockerfile_path}/Dockerfile" + if not exists(dockerfile): + continue + with open(dockerfile, 'r') as file: + if re.search(gunicorn_pattern, file.read()): + return candidate + requirements = f"{candidate}/../requirements.txt" + if not exists(requirements): + continue + with open(requirements, 'r') as file: + if re.search(gunicorn_pattern, file.read()): return candidate return None - flask_main = identify_flask_main(mains_candidates) + task_main_file = identify_unicorn_based_main(mains_candidates) - if flask_main: + if task_main_file: release_config['overrides']['apps'][app_key] = \ { 'harness': { 'deployment': { 'command': ['python'], - 'args': [f'/usr/src/app/{os.path.basename(flask_main)}/__main__.py'] + 'args': [f'/usr/src/app/{os.path.basename(task_main_file)}/__main__.py'] } } } diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile index 5ff5b821..533a9c95 100644 --- a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile @@ -1,2 +1,4 @@ ARG CLOUDHARNESS_FLASK -FROM $CLOUDHARNESS_FLASK \ No newline at end of file +FROM $CLOUDHARNESS_FLASK + +RUN pip install gunicorn \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore new file mode 100644 index 00000000..669be813 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore @@ -0,0 +1 @@ +*.ignored \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile new file mode 100644 index 00000000..de89f0ed --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile @@ -0,0 +1,4 @@ +ARG CLOUDHARNESS_FLASK +FROM $CLOUDHARNESS_FLASK + +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/resources/aresource.txt b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/resources/aresource.txt new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/templates/mytemplate.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/templates/mytemplate.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml new file mode 100644 index 00000000..0b7c9a77 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml @@ -0,0 +1,11 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + - my-common +a: b +dev: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml new file mode 100644 index 00000000..85ecbfae --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml @@ -0,0 +1,2 @@ +a: test +test: true \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml new file mode 100644 index 00000000..f3519b50 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml @@ -0,0 +1,4 @@ +harness: + database: + auto: true + type: mongo \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withoutdb.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withoutdb.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml new file mode 100644 index 00000000..4b260dbf --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml @@ -0,0 +1,2 @@ +harness: + database: {auto: true, type: postgres} \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml new file mode 100644 index 00000000..434b042b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml @@ -0,0 +1,14 @@ +harness: + name: "I'm useless" + subdomain: mysubdomain + dependencies: + soft: + - legacy + build: + - cloudharness-flask + test: + unit: + commands: + - tox + - echo "hello" +a: b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py new file mode 100644 index 00000000..3118318a --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +from cloudharness.utils.server import init_flask, main + + +app = init_flask(title="Cloudharness sample application", webapp=True) + +if __name__ == '__main__': + main() diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt new file mode 100644 index 00000000..283e1305 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt @@ -0,0 +1 @@ +gunicorn \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/tasks/mytask/Dockerfile b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/tasks/mytask/Dockerfile new file mode 100644 index 00000000..e69de29b diff --git a/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py new file mode 100644 index 00000000..37de2b0b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from something import something_else + + +def fake_content(): + ... \ No newline at end of file diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index f8b7b659..94998892 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -139,3 +139,39 @@ def test_create_skaffold_configuration_with_conflicting_dependencies(tmp_path): myapp_config = release['overrides']['apps']['myapp'] assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' + + +def test_create_skaffold_configuration_with_conflicting_dependencies_requirements_file(tmp_path): + values = create_helm_chart( + [CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + output_path=OUT, + include=['myapp2'], + exclude=['events'], + domain="my.local", + namespace='test', + env='dev', + local=False, + tag=1, + registry='reg' + ) + root_paths = preprocess_build_overrides( + root_paths=[CLOUDHARNESS_ROOT, RESOURCES_BUGGY], + helm_values=values, + merge_build_path=str(tmp_path) + ) + + sk = create_skaffold_configuration( + root_paths=root_paths, + helm_values=values, + output_path=OUT + ) + + releases = sk['deploy']['helm']['releases'] + assert len(releases) == 1 # Ensure we only found 1 deployment (for myapp) + + release = releases[0] + assert 'myapp2' in release['overrides']['apps'] + assert 'matplotlib' not in release['overrides']['apps'] + + myapp_config = release['overrides']['apps']['myapp2'] + assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py' \ No newline at end of file