diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-app/api/templates/main.jinja2 index e351e841f..2afff27a6 100644 --- a/application-templates/django-app/api/templates/main.jinja2 +++ b/application-templates/django-app/api/templates/main.jinja2 @@ -14,13 +14,13 @@ from fastapi.staticfiles import StaticFiles {{imports | replace(".","openapi.")}} -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") apps.populate(settings.INSTALLED_APPS) # migrate the Django models os.system("python manage.py migrate") -from api.controllers import * +from __APP_NAME__.controllers import * app = FastAPI( {% if info %} @@ -66,10 +66,12 @@ async def add_process_time_header(request: Request, call_next): if os.environ.get('KUBERNETES_SERVICE_HOST', None): # init the auth service when running in/for k8s - from cloudharness_django.services import init_services, get_auth_service - init_services() + from cloudharness_django.services import init_services_in_background, get_auth_service + init_services_in_background() + # start the kafka event listener when running in/for k8s - import cloudharness_django.services.events + from cloudharness_django.services.events import init_listener_in_background + init_listener_in_background() async def has_access(): """ diff --git a/application-templates/django-app/backend/api/admin.py b/application-templates/django-app/backend/__APP_NAME__/admin.py similarity index 100% rename from application-templates/django-app/backend/api/admin.py rename to application-templates/django-app/backend/__APP_NAME__/admin.py diff --git a/application-templates/django-app/backend/api/apps.py b/application-templates/django-app/backend/__APP_NAME__/apps.py similarity index 82% rename from application-templates/django-app/backend/api/apps.py rename to application-templates/django-app/backend/__APP_NAME__/apps.py index 66656fd29..456a2f40d 100644 --- a/application-templates/django-app/backend/api/apps.py +++ b/application-templates/django-app/backend/__APP_NAME__/apps.py @@ -3,4 +3,4 @@ class ApiConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'api' + name = '__APP_NAME__' diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py b/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py new file mode 100644 index 000000000..71fac3207 --- /dev/null +++ b/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py @@ -0,0 +1 @@ +import __APP_NAME__.controllers.test as test_controller diff --git a/application-templates/django-app/backend/api/controllers/test.py b/application-templates/django-app/backend/__APP_NAME__/controllers/test.py similarity index 100% rename from application-templates/django-app/backend/api/controllers/test.py rename to application-templates/django-app/backend/__APP_NAME__/controllers/test.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py index da7c00b1a..8120bb654 100644 --- a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py +++ b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py @@ -1,24 +1,10 @@ -import os - from django.db import migrations -def create_kc_client_and_roles(apps, schema_editor): - if os.environ.get("KUBERNETES_SERVICE_HOST", None): - # running in K8S so create the KC client and roles - from cloudharness_django.services import get_auth_service, get_user_service, init_services - - init_services() - get_auth_service().create_client() - get_user_service().sync_kc_users_groups() - - class Migration(migrations.Migration): dependencies = [ - ("cloudharness_django", "0001_initial"), + ("django_baseapp", "0001_initial"), ] - operations = [ - migrations.RunPython(create_kc_client_and_roles), - ] + operations = [] diff --git a/application-templates/django-app/backend/api/tests.py b/application-templates/django-app/backend/__APP_NAME__/tests.py similarity index 100% rename from application-templates/django-app/backend/api/tests.py rename to application-templates/django-app/backend/__APP_NAME__/tests.py diff --git a/application-templates/django-app/backend/__APP_NAME__/views.py b/application-templates/django-app/backend/__APP_NAME__/views.py index b9a7c360c..91ea44a21 100644 --- a/application-templates/django-app/backend/__APP_NAME__/views.py +++ b/application-templates/django-app/backend/__APP_NAME__/views.py @@ -1,24 +1,3 @@ -import mimetypes -from pathlib import Path +from django.shortcuts import render -from django.conf import settings -from django.http import FileResponse, HttpResponseRedirect -from django.urls import reverse -from django.utils._os import safe_join - - -def view_404(request, exception=None): - return HttpResponseRedirect(reverse("index")) - - -def index(request, path=""): - if path == "": - path = "index.html" - fullpath = Path(safe_join(settings.STATIC_ROOT, "www", path)) - content_type, encoding = mimetypes.guess_type(str(fullpath)) - content_type = content_type or "application/octet-stream" - try: - fullpath.open("rb") - except FileNotFoundError: - return index(request, "") # index.html - return FileResponse(fullpath.open("rb"), content_type=content_type) +# Create your views here. diff --git a/application-templates/django-app/backend/api/controllers/__init__.py b/application-templates/django-app/backend/api/controllers/__init__.py deleted file mode 100644 index 1344f5cc9..000000000 --- a/application-templates/django-app/backend/api/controllers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import api.controllers.test as test_controller diff --git a/application-templates/django-app/backend/api/views.py b/application-templates/django-app/backend/api/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/application-templates/django-app/backend/api/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/application-templates/django-app/backend/api/__init__.py b/application-templates/django-app/backend/django_baseapp/__init__.py similarity index 100% rename from application-templates/django-app/backend/api/__init__.py rename to application-templates/django-app/backend/django_baseapp/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/asgi.py b/application-templates/django-app/backend/django_baseapp/asgi.py similarity index 85% rename from application-templates/django-app/backend/__APP_NAME__/asgi.py rename to application-templates/django-app/backend/django_baseapp/asgi.py index 6d1bf4d96..049df40e6 100644 --- a/application-templates/django-app/backend/__APP_NAME__/asgi.py +++ b/application-templates/django-app/backend/django_baseapp/asgi.py @@ -21,4 +21,6 @@ init_services() # start the kafka event listener -import cloudharness_django.services.events # noqa E402 +from cloudharness_django.services.events import init_listner # noqa E402 + +init_listner() diff --git a/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py b/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py new file mode 100644 index 000000000..da7c00b1a --- /dev/null +++ b/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py @@ -0,0 +1,24 @@ +import os + +from django.db import migrations + + +def create_kc_client_and_roles(apps, schema_editor): + if os.environ.get("KUBERNETES_SERVICE_HOST", None): + # running in K8S so create the KC client and roles + from cloudharness_django.services import get_auth_service, get_user_service, init_services + + init_services() + get_auth_service().create_client() + get_user_service().sync_kc_users_groups() + + +class Migration(migrations.Migration): + + dependencies = [ + ("cloudharness_django", "0001_initial"), + ] + + operations = [ + migrations.RunPython(create_kc_client_and_roles), + ] diff --git a/application-templates/django-app/backend/api/migrations/__init__.py b/application-templates/django-app/backend/django_baseapp/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/api/migrations/__init__.py rename to application-templates/django-app/backend/django_baseapp/migrations/__init__.py diff --git a/application-templates/django-app/backend/api/models.py b/application-templates/django-app/backend/django_baseapp/models.py similarity index 100% rename from application-templates/django-app/backend/api/models.py rename to application-templates/django-app/backend/django_baseapp/models.py diff --git a/application-templates/django-app/backend/__APP_NAME__/settings.py b/application-templates/django-app/backend/django_baseapp/settings.py similarity index 97% rename from application-templates/django-app/backend/__APP_NAME__/settings.py rename to application-templates/django-app/backend/django_baseapp/settings.py index 9b3c6d221..78ba0cbc9 100644 --- a/application-templates/django-app/backend/__APP_NAME__/settings.py +++ b/application-templates/django-app/backend/django_baseapp/settings.py @@ -53,7 +53,7 @@ ] -ROOT_URLCONF = "__APP_NAME__.urls" +ROOT_URLCONF = "django_baseapp.urls" TEMPLATES = [ { @@ -71,7 +71,7 @@ }, ] -WSGI_APPLICATION = "__APP_NAME__.wsgi.application" +WSGI_APPLICATION = "django_baseapp.wsgi.application" # Password validation @@ -130,8 +130,8 @@ # add the local apps INSTALLED_APPS += [ - "api", - "__APP_NAME__" + "__APP_NAME__", + "django_baseapp" ] # override django admin base template with a local template diff --git a/application-templates/django-app/backend/__APP_NAME__/static/www/index.html b/application-templates/django-app/backend/django_baseapp/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/static/www/index.html rename to application-templates/django-app/backend/django_baseapp/static/www/index.html diff --git a/application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/index.html b/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/index.html rename to application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/__APP_NAME__/urls.py b/application-templates/django-app/backend/django_baseapp/urls.py similarity index 96% rename from application-templates/django-app/backend/__APP_NAME__/urls.py rename to application-templates/django-app/backend/django_baseapp/urls.py index ce1b9629e..ecfee23ee 100644 --- a/application-templates/django-app/backend/__APP_NAME__/urls.py +++ b/application-templates/django-app/backend/django_baseapp/urls.py @@ -19,7 +19,7 @@ from django.contrib import admin from django.urls import path, re_path -from __APP_NAME__.views import index +from django_baseapp.views import index urlpatterns = [path("admin/", admin.site.urls)] diff --git a/application-templates/django-app/backend/django_baseapp/views.py b/application-templates/django-app/backend/django_baseapp/views.py new file mode 100644 index 000000000..b9a7c360c --- /dev/null +++ b/application-templates/django-app/backend/django_baseapp/views.py @@ -0,0 +1,24 @@ +import mimetypes +from pathlib import Path + +from django.conf import settings +from django.http import FileResponse, HttpResponseRedirect +from django.urls import reverse +from django.utils._os import safe_join + + +def view_404(request, exception=None): + return HttpResponseRedirect(reverse("index")) + + +def index(request, path=""): + if path == "": + path = "index.html" + fullpath = Path(safe_join(settings.STATIC_ROOT, "www", path)) + content_type, encoding = mimetypes.guess_type(str(fullpath)) + content_type = content_type or "application/octet-stream" + try: + fullpath.open("rb") + except FileNotFoundError: + return index(request, "") # index.html + return FileResponse(fullpath.open("rb"), content_type=content_type) diff --git a/application-templates/django-app/backend/__APP_NAME__/wsgi.py b/application-templates/django-app/backend/django_baseapp/wsgi.py similarity index 74% rename from application-templates/django-app/backend/__APP_NAME__/wsgi.py rename to application-templates/django-app/backend/django_baseapp/wsgi.py index 678932f0f..e2d6fc099 100644 --- a/application-templates/django-app/backend/__APP_NAME__/wsgi.py +++ b/application-templates/django-app/backend/django_baseapp/wsgi.py @@ -11,7 +11,7 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") application = get_wsgi_application() @@ -21,4 +21,6 @@ init_services() # start the kafka event listener -import cloudharness_django.services.events # noqa E402 +from cloudharness_django.services.events import init_listner # noqa E402 + +init_listner() diff --git a/application-templates/django-app/backend/manage.py b/application-templates/django-app/backend/manage.py index 0412a721a..d106b9543 100644 --- a/application-templates/django-app/backend/manage.py +++ b/application-templates/django-app/backend/manage.py @@ -6,7 +6,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/application-templates/django-app/dev-setup.sh b/application-templates/django-app/dev-setup.sh new file mode 100644 index 000000000..7bdd82505 --- /dev/null +++ b/application-templates/django-app/dev-setup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$(pwd) +CH_DIRECTORY="../../cloud-harness" +INSTALL_PYTEST=false +CURRENT_DIRECTORY="$(pwd)" +APP_NAME="__APP_NAME__" + +pip_upgrade_error() { + echo "Unable to upgrade pip" + exit 1 +} + +install_error () { + echo "Unable to install $1" 1>&2 + exit 1 +} + +while getopts ch_directory:pytest arg; +do + case "$arg" in + ch_directory) CH_DIRECTORY=${OPTARG};; + pytest) INSTALL_PYTEST=true;; + esac +done + +pip install --upgrade pip || pip_upgrade_error + +# Install pip dependencies from cloudharness-base-debian image + +if $INSTALL_PYTEST; then + pip install pytest || install_error pytest +fi + +pip install -r "$CH_DIRECTORY/libraries/models/requirements.txt" || install_error "models requirements" +pip install -r "$CH_DIRECTORY/libraries/cloudharness-common/requirements.txt" || install_error "cloudharness-common requirements" +pip install -r "$CH_DIRECTORY/libraries/client/cloudharness_cli/requirements.txt" || install_error "cloudharness_cli requirements" + +pip install -e "$CH_DIRECTORY/libraries/models" || install_error models +pip install -e "$CH_DIRECTORY/libraries/cloudharness-common" || install_error cloudharness-common +pip install -e "$CH_DIRECTORY/libraries/client/cloudharness_cli" || install_error cloudharness_cli + +# Install pip dependencies from cloudharness-django image + +pip install -r "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/fastapi/requirements.txt" || install_error "cloudharness-django fastapi requirements" +pip install -e "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django" || install_error cloudharness-django + +# Install application + +pip install -r "$CURRENT_DIRECTORY/backend/requirements.txt" || install_error "$APP_NAME dependencies" +pip install -e "$CURRENT_DIRECTORY/backend" || install_error "$APP_NAME" \ No newline at end of file diff --git a/deployment-configuration/vscode-django-app-debug-template.json b/deployment-configuration/vscode-django-app-debug-template.json new file mode 100644 index 000000000..f049d5e53 --- /dev/null +++ b/deployment-configuration/vscode-django-app-debug-template.json @@ -0,0 +1,24 @@ +{ + "args": [ + "--host", + "0.0.0.0", + "--port", + "8000", + "main:app" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/applications/__APP_NAME__/backend", + "env": { + "ACCOUNTS_ADMIN_PASSWORD": "metacell", + "ACCOUNTS_ADMIN_USERNAME": "admin", + "CH_CURRENT_APP_NAME": "__APP_NAME__", + "CH_VALUES_PATH": "${workspaceFolder}/deployment/helm/values.yaml", + "DJANGO_SETTINGS_MODULE": "django_baseapp.settings", + "KUBERNETES_SERVICE_HOST": "ssdds" + }, + "justMyCode": false, + "module": "uvicorn", + "name": "__APP_NAME__ backend", + "request": "launch", + "type": "debugpy" + } \ No newline at end of file diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md index e6d7074bd..9e4eda591 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md @@ -46,7 +46,9 @@ Quick start init_services() # start the kafka event listener - import cloudharness_django.services.events + from cloudharness_django.services.events import init_listner # noqa E402 + + init_listner() ``` 4. Start the development server and visit http://127.0.0.1:8000/ diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py index 4efca5168..330a760eb 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py @@ -43,3 +43,28 @@ def init_services( admin_role=admin_role) _user_service = UserService(_auth_service) return _auth_service + + +def init_services_in_background( + client_name: str = settings.KC_CLIENT_NAME, + client_roles: List[str] = settings.KC_ALL_ROLES, + privileged_roles: List[str] = settings.KC_PRIVILEGED_ROLES, + admin_role: str = settings.KC_ADMIN_ROLE, + default_user_role: str = settings.KC_DEFAULT_USER_ROLE +): + import threading + import time + from cloudharness import log + + def background_operation(): + services_initialized = False + + while not services_initialized: + try: + init_services(client_name, client_roles, privileged_roles, admin_role, default_user_role) + services_initialized = True + except: + log.exception("Error initializing services. Retrying in 5 seconds...") + time.sleep(5) + + threading.Thread(target=background_operation).start() diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index 55909afe2..170b41ac0 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -15,6 +15,7 @@ class KeycloakMessageService: def __init__(self, kafka_group_id): self._topic = "keycloak.fct.admin" self.kafka_group_id = kafka_group_id + self.topics_initialized = False @staticmethod def event_handler(app, event_client, message): @@ -50,6 +51,9 @@ def event_handler(app, event_client, message): raise e def init_topics(self): + if self.topics_initialized: + return + log.info("Starting Kafka consumer threads") try: event_client = EventClient(self._topic) @@ -59,6 +63,7 @@ def init_topics(self): except TopicAlreadyExistsError as e: pass event_client.async_consume(app={}, group_id=self.kafka_group_id, handler=KeycloakMessageService.event_handler) + self.topics_initialized = True except Exception as e: log.error(f"Error creating topic {self._topic}", exc_info=e) @@ -79,8 +84,35 @@ def setup_event_service(self): pass -# start services -if not hasattr(settings, "PROJECT_NAME"): - raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") +_message_service_singleton = None + + +def init_listener(): + if not hasattr(settings, "PROJECT_NAME"): + raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") + + global _message_service_singleton + if _message_service_singleton is None: + _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME) + + _message_service_singleton.setup_event_service() -KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() + +def init_listener_in_background(): + import threading + import time + from cloudharness import log + + def background_operation(): + listener_initialized = False + + while not listener_initialized: + try: + init_listener() + log.info('User sync events listener started') + listener_initialized = True + except: + log.exception('Error initializing event queue. Retrying in 5 seconds...') + time.sleep(5) + + threading.Thread(target=background_operation).start() diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg index d298dece8..5ca2ead8d 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg @@ -31,6 +31,6 @@ python_requires = >=3.6 install_requires = Django>=4.0.7 django-admin-extra-buttons>=1.4.2 - psycopg2-binary==2.9.3 - Pillow==9.2.0 + psycopg2-binary>=2.9.3 + Pillow>=9.2.0 python-keycloak diff --git a/libraries/cloudharness-common/cloudharness/middleware/django.py b/libraries/cloudharness-common/cloudharness/middleware/django.py index e78837511..f3f6ee760 100644 --- a/libraries/cloudharness-common/cloudharness/middleware/django.py +++ b/libraries/cloudharness-common/cloudharness/middleware/django.py @@ -1,6 +1,7 @@ from cloudharness.middleware import set_authentication_token from django.http.request import HttpRequest + class CloudharnessMiddleware: def __init__(self, get_response): self.get_response = get_response diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index e6f232234..4df7b8834 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -33,7 +33,7 @@ def generate_server(app_path, overrides_folder=""): def generate_fastapi_server(app_path): - command = f"cd {app_path}/api && bash -c ./genapi.sh" + command = f"cd {app_path}/api && bash genapi.sh" os.system(command) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 7a00626d9..85fcb5bba 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -2,6 +2,7 @@ import socket import glob import subprocess +from typing import Any import requests import os from functools import cache @@ -188,6 +189,22 @@ def replace_in_file(src_file, source, replace): pass +def replace_in_dict(src_dict: dict, source: str, replacement: str) -> dict: + def replace_value(value: Any) -> Any: + if isinstance(value, str): + return value.replace(source, replacement) + if isinstance(value, list): + return [replace_value(item) for item in value] + if isinstance(value, dict): + return replace_in_dict(value, source, replacement) + return value + + return { + key: replace_value(value) + for key, value in src_dict.items() + } + + def copymergedir(root_src_dir, root_dst_dir): """ Does copy and merge (shutil.copytree requires that the destination does not exist) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 551d0ae1d..d3601f7a5 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -1,31 +1,86 @@ #!/usr/bin/env python +import json +import pathlib import sys import os import re +import shutil import tempfile import subprocess import logging +import argparse from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ - to_python_module, copymergedir + to_python_module, copymergedir, get_json_template, replace_in_dict + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser( - description='Creates a new Application.') +class TemplateType(StrEnum): + BASE = 'base' + FLASK_SERVER = 'flask-server' + WEBAPP = 'webapp' + DB_POSTGRES = 'db-postgres' + DB_NEO4J = 'db-neo4j' + DB_MONGO = 'db-mongo' + DJANGO_APP = 'django-app' + SERVER = 'server' + + +def main(): + app_name, templates = get_command_line_arguments() + + app_path = os.path.join(APPLICATIONS_SRC_PATH, app_name) + os.makedirs(app_path, exist_ok=True) + + if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: + templates = [TemplateType.BASE, TemplateType.WEBAPP] + templates + + if TemplateType.WEBAPP in templates: + handle_webapp_template(app_name, app_path) + + if TemplateType.SERVER in templates: + handle_server_template(app_path) + + for template_name in templates: + merge_template_directories(template_name, app_path) + + if TemplateType.FLASK_SERVER in templates: + handle_flask_server_template(app_path) + + replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(app_name)) + + if TemplateType.DJANGO_APP in templates: + handle_django_app_template(app_name, app_path) + + replaceindir(app_path, PLACEHOLDER, app_name) + + if TemplateType.WEBAPP in templates: + handle_webapp_template_cleanup(app_path) + + +def get_command_line_arguments() -> tuple[str, list[str]]: + parser = argparse.ArgumentParser(description='Creates a new Application.') + parser.add_argument('name', metavar='name', type=str, help='Application name') - parser.add_argument('-t', '--template', dest='templates', action="append", default=['base', ], + parser.add_argument('-t', '--template', + dest='templates', + action="append", + default=[TemplateType.BASE], + type=str, help="""Add a template name. Available templates: @@ -54,58 +109,82 @@ if __name__ == "__main__": print("Invalid regex") exit(1) - app_path = os.path.join(APPLICATIONS_SRC_PATH, args.name) - os.makedirs(app_path, exist_ok=True) - templates = args.templates - - if "django-app" in args.templates and "webapp" not in templates: - templates = ["base", "webapp"] + templates - - if 'webapp' in templates: - if os.path.exists(os.path.join(app_path, 'frontend')): - shutil.rmtree(os.path.join(app_path, 'frontend')) - cmd = ["yarn", "create", "vite", args.name, "--template", "react-ts"] - logging.info(f"Running command: {' '.join(cmd)}") - subprocess.run(cmd, cwd=app_path) - shutil.move(os.path.join(app_path, args.name), os.path.join(app_path, 'frontend')) - generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) - - if 'server' in templates: - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) + return args.name, args.templates - for template_name in templates: - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) - - if "flask-server" in templates: - generate_server(app_path) - - replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(args.name)) - - if "django-app" in templates: - replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(args.name)) - generate_fastapi_server(app_path) - replace_in_file( - os.path.join(app_path, 'deploy/values.yaml'), - f"{PLACEHOLDER}:{PLACEHOLDER}", - f"{to_python_module(args.name)}:{to_python_module(args.name)}" - ) - try: - os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) - except FileNotFoundError: - # backend dockerfile not found, continue - pass - - replaceindir(app_path, PLACEHOLDER, args.name) - - if 'webapp' in templates: - try: - os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) - except FileNotFoundError: - # backend dockerfile not found, continue - pass + +def handle_webapp_template(app_name: str, app_path: str) -> None: + if os.path.exists(os.path.join(app_path, 'frontend')): + shutil.rmtree(os.path.join(app_path, 'frontend')) + cmd = ["yarn", "create", "vite", app_name, "--template", "react-ts"] + logging.info(f"Running command: {' '.join(cmd)}") + subprocess.run(cmd, cwd=app_path) + shutil.move(os.path.join(app_path, app_name), os.path.join(app_path, 'frontend')) + generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) + + +def handle_webapp_template_cleanup(app_path: str) -> None: + try: + os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) + except FileNotFoundError: + # backend dockerfile not found, continue + pass + + +def handle_server_template(app_path: str) -> None: + with tempfile.TemporaryDirectory() as tmp_dirname: + copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, TemplateType.SERVER), tmp_dirname) + merge_configuration_directories(app_path, tmp_dirname) + generate_server(app_path, tmp_dirname) + + +def handle_flask_server_template(app_path: str) -> None: + generate_server(app_path) + + +def handle_django_app_template(app_name: str, app_path: str) -> None: + replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(app_name)) + generate_fastapi_server(app_path) + replace_in_file( + os.path.join(app_path, 'deploy/values.yaml'), + f"{PLACEHOLDER}:{PLACEHOLDER}", + f"{to_python_module(app_name)}:{to_python_module(app_name)}" + ) + replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) + create_django_app_vscode_debug_configuration(app_name) + try: + os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) + except FileNotFoundError: + # backend dockerfile not found, continue + pass + + +def create_django_app_vscode_debug_configuration(app_name: str): + vscode_launch_path = pathlib.Path('.vscode/launch.json') + configuration_name = f'{app_name} backend' + + launch_config = get_json_template(vscode_launch_path, True) + + launch_config['configurations'] = [ + configuration for configuration in launch_config['configurations'] + if configuration['name'] != configuration_name + ] + + debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = replace_in_dict(debug_config, PLACEHOLDER, app_name) + + launch_config['configurations'].append(debug_config) + + vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) + with vscode_launch_path.open('w') as f: + json.dump(launch_config, f, indent=2, sort_keys=True) + + +def merge_template_directories(template_name: str, app_path: str) -> None: + for base_path in (CH_ROOT, os.getcwd()): + template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) + if os.path.exists(template_path): + merge_configuration_directories(template_path, app_path) + + +if __name__ == "__main__": + main() diff --git a/tools/deployment-cli-tools/requirements.txt b/tools/deployment-cli-tools/requirements.txt index 622565b9a..266ae27f3 100644 --- a/tools/deployment-cli-tools/requirements.txt +++ b/tools/deployment-cli-tools/requirements.txt @@ -4,4 +4,5 @@ ruamel.yaml oyaml cloudharness_model cloudharness_utils -dirhash \ No newline at end of file +dirhash +StrEnum ; python_version < '3.11' \ No newline at end of file diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index b01b2ae24..280cd716b 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -28,10 +28,10 @@ 'cloudharness_model', 'cloudharness_utils', 'fastapi-code-generator', - 'dirhash' + 'dirhash', + "StrEnum ; python_version < '3.11'", ] - setup( name=NAME, version=VERSION, diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 78e843334..0c4060f36 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -87,3 +87,64 @@ def test_find_dockerfile_paths(): assert len(dockerfiles) == 2 assert next(d for d in dockerfiles if d.endswith("myapp")), "Must find the Dockerfile in the root directory" assert next(d for d in dockerfiles if d.endswith("myapp/tasks/mytask")), "Must find the Dockerfile in the tasks directory" + + +class TestReplaceInDict: + def test_does_not_replace_in_keys(_): + src_dict = { + 'foo': 1, + 'bar': 2, + 'baz': 3, + 'foobar': 4, + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict.keys() == src_dict.keys() + + def test_replaces_in_values(_): + src_dict = { + 'a': 'foo', + 'b': 'bar', + 'c': 'baz', + 'd': 3, + 'e': 'foobar', + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict == { + 'a': 'xxx', + 'b': 'bar', + 'c': 'baz', + 'd': 3, + 'e': 'xxxbar', + } + + def test_replaces_in_values_within_lists(_): + src_dict = { + 'a': ['foo', 'bar', 'baz', 3, 'foobar'], + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict['a'] == ['xxx', 'bar', 'baz', 3, 'xxxbar'] + + def test_replaces_in_values_within_nested_dict(_): + src_dict = { + 'a': { + 'a': 'foo', + 'b': 'bar', + 'c': 'foobar', + 'e': ['foo', 'bar', 'foobar'], + }, + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict['a'] == { + 'a': 'xxx', + 'b': 'bar', + 'c': 'xxxbar', + 'e': ['xxx', 'bar', 'xxxbar'] + }