From 29f6dd39124d9d1e9a2da54a80ae7860f2e43a31 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 12 Sep 2024 11:51:48 +0100 Subject: [PATCH 01/21] CH-59 Rename __APP_NAME__ directory to ch_django and api directory to __APP_NAME__ --- .../backend/{api => __APP_NAME__}/admin.py | 0 .../backend/{api => __APP_NAME__}/apps.py | 0 .../controllers/__init__.py | 0 .../{api => __APP_NAME__}/controllers/test.py | 0 .../backend/{api => __APP_NAME__}/tests.py | 0 .../django-app/backend/__APP_NAME__/views.py | 25 ++----------------- .../django-app/backend/api/views.py | 3 --- .../backend/{api => ch_django}/__init__.py | 0 .../{__APP_NAME__ => ch_django}/asgi.py | 0 .../migrations/0001_initial.py | 0 .../{api => ch_django}/migrations/__init__.py | 0 .../backend/{api => ch_django}/models.py | 0 .../{__APP_NAME__ => ch_django}/settings.py | 0 .../static/www/index.html | 0 .../templates/__APP_NAME__/index.html | 0 .../templates/__APP_NAME__/swagger-ui.html | 0 .../{__APP_NAME__ => ch_django}/urls.py | 0 .../django-app/backend/ch_django/views.py | 24 ++++++++++++++++++ .../{__APP_NAME__ => ch_django}/wsgi.py | 0 19 files changed, 26 insertions(+), 26 deletions(-) rename application-templates/django-app/backend/{api => __APP_NAME__}/admin.py (100%) rename application-templates/django-app/backend/{api => __APP_NAME__}/apps.py (100%) rename application-templates/django-app/backend/{api => __APP_NAME__}/controllers/__init__.py (100%) rename application-templates/django-app/backend/{api => __APP_NAME__}/controllers/test.py (100%) rename application-templates/django-app/backend/{api => __APP_NAME__}/tests.py (100%) delete mode 100644 application-templates/django-app/backend/api/views.py rename application-templates/django-app/backend/{api => ch_django}/__init__.py (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/asgi.py (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/migrations/0001_initial.py (100%) rename application-templates/django-app/backend/{api => ch_django}/migrations/__init__.py (100%) rename application-templates/django-app/backend/{api => ch_django}/models.py (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/settings.py (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/static/www/index.html (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/templates/__APP_NAME__/index.html (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/templates/__APP_NAME__/swagger-ui.html (100%) rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/urls.py (100%) create mode 100644 application-templates/django-app/backend/ch_django/views.py rename application-templates/django-app/backend/{__APP_NAME__ => ch_django}/wsgi.py (100%) diff --git a/application-templates/django-app/backend/api/admin.py b/application-templates/django-app/backend/__APP_NAME__/admin.py similarity index 100% rename from application-templates/django-app/backend/api/admin.py rename to application-templates/django-app/backend/__APP_NAME__/admin.py diff --git a/application-templates/django-app/backend/api/apps.py b/application-templates/django-app/backend/__APP_NAME__/apps.py similarity index 100% rename from application-templates/django-app/backend/api/apps.py rename to application-templates/django-app/backend/__APP_NAME__/apps.py diff --git a/application-templates/django-app/backend/api/controllers/__init__.py b/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py similarity index 100% rename from application-templates/django-app/backend/api/controllers/__init__.py rename to application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py diff --git a/application-templates/django-app/backend/api/controllers/test.py b/application-templates/django-app/backend/__APP_NAME__/controllers/test.py similarity index 100% rename from application-templates/django-app/backend/api/controllers/test.py rename to application-templates/django-app/backend/__APP_NAME__/controllers/test.py diff --git a/application-templates/django-app/backend/api/tests.py b/application-templates/django-app/backend/__APP_NAME__/tests.py similarity index 100% rename from application-templates/django-app/backend/api/tests.py rename to application-templates/django-app/backend/__APP_NAME__/tests.py diff --git a/application-templates/django-app/backend/__APP_NAME__/views.py b/application-templates/django-app/backend/__APP_NAME__/views.py index b9a7c360c..91ea44a21 100644 --- a/application-templates/django-app/backend/__APP_NAME__/views.py +++ b/application-templates/django-app/backend/__APP_NAME__/views.py @@ -1,24 +1,3 @@ -import mimetypes -from pathlib import Path +from django.shortcuts import render -from django.conf import settings -from django.http import FileResponse, HttpResponseRedirect -from django.urls import reverse -from django.utils._os import safe_join - - -def view_404(request, exception=None): - return HttpResponseRedirect(reverse("index")) - - -def index(request, path=""): - if path == "": - path = "index.html" - fullpath = Path(safe_join(settings.STATIC_ROOT, "www", path)) - content_type, encoding = mimetypes.guess_type(str(fullpath)) - content_type = content_type or "application/octet-stream" - try: - fullpath.open("rb") - except FileNotFoundError: - return index(request, "") # index.html - return FileResponse(fullpath.open("rb"), content_type=content_type) +# Create your views here. diff --git a/application-templates/django-app/backend/api/views.py b/application-templates/django-app/backend/api/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/application-templates/django-app/backend/api/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/application-templates/django-app/backend/api/__init__.py b/application-templates/django-app/backend/ch_django/__init__.py similarity index 100% rename from application-templates/django-app/backend/api/__init__.py rename to application-templates/django-app/backend/ch_django/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/asgi.py b/application-templates/django-app/backend/ch_django/asgi.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/asgi.py rename to application-templates/django-app/backend/ch_django/asgi.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-app/backend/ch_django/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py rename to application-templates/django-app/backend/ch_django/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/api/migrations/__init__.py b/application-templates/django-app/backend/ch_django/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/api/migrations/__init__.py rename to application-templates/django-app/backend/ch_django/migrations/__init__.py diff --git a/application-templates/django-app/backend/api/models.py b/application-templates/django-app/backend/ch_django/models.py similarity index 100% rename from application-templates/django-app/backend/api/models.py rename to application-templates/django-app/backend/ch_django/models.py diff --git a/application-templates/django-app/backend/__APP_NAME__/settings.py b/application-templates/django-app/backend/ch_django/settings.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/settings.py rename to application-templates/django-app/backend/ch_django/settings.py diff --git a/application-templates/django-app/backend/__APP_NAME__/static/www/index.html b/application-templates/django-app/backend/ch_django/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/static/www/index.html rename to application-templates/django-app/backend/ch_django/static/www/index.html diff --git a/application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/index.html b/application-templates/django-app/backend/ch_django/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/index.html rename to application-templates/django-app/backend/ch_django/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-app/backend/ch_django/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-app/backend/ch_django/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/__APP_NAME__/urls.py b/application-templates/django-app/backend/ch_django/urls.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/urls.py rename to application-templates/django-app/backend/ch_django/urls.py diff --git a/application-templates/django-app/backend/ch_django/views.py b/application-templates/django-app/backend/ch_django/views.py new file mode 100644 index 000000000..b9a7c360c --- /dev/null +++ b/application-templates/django-app/backend/ch_django/views.py @@ -0,0 +1,24 @@ +import mimetypes +from pathlib import Path + +from django.conf import settings +from django.http import FileResponse, HttpResponseRedirect +from django.urls import reverse +from django.utils._os import safe_join + + +def view_404(request, exception=None): + return HttpResponseRedirect(reverse("index")) + + +def index(request, path=""): + if path == "": + path = "index.html" + fullpath = Path(safe_join(settings.STATIC_ROOT, "www", path)) + content_type, encoding = mimetypes.guess_type(str(fullpath)) + content_type = content_type or "application/octet-stream" + try: + fullpath.open("rb") + except FileNotFoundError: + return index(request, "") # index.html + return FileResponse(fullpath.open("rb"), content_type=content_type) diff --git a/application-templates/django-app/backend/__APP_NAME__/wsgi.py b/application-templates/django-app/backend/ch_django/wsgi.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/wsgi.py rename to application-templates/django-app/backend/ch_django/wsgi.py From 9127e92d2082b016ab23cfdedf26a0df601a0f47 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 12 Sep 2024 11:53:53 +0100 Subject: [PATCH 02/21] CH-59 Move settings back to __APP_NAME__ directory --- .../django-app/backend/{ch_django => __APP_NAME__}/settings.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename application-templates/django-app/backend/{ch_django => __APP_NAME__}/settings.py (100%) diff --git a/application-templates/django-app/backend/ch_django/settings.py b/application-templates/django-app/backend/__APP_NAME__/settings.py similarity index 100% rename from application-templates/django-app/backend/ch_django/settings.py rename to application-templates/django-app/backend/__APP_NAME__/settings.py From 301b8edb2ad46a57f5c739e1b56ca8f60ab16184 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 12 Sep 2024 12:03:39 +0100 Subject: [PATCH 03/21] CH-59 Add initial api migration depending on ch_django migration --- .../backend/__APP_NAME__/migrations/0001_initial.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py new file mode 100644 index 000000000..34c1f836d --- /dev/null +++ b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py @@ -0,0 +1,10 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("ch_django", "0001_initial"), + ] + + operations = [] \ No newline at end of file From 15b447080676e295933beadff2c1a73200bfcd0f Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 13 Sep 2024 14:31:19 +0100 Subject: [PATCH 04/21] CH-59 Update api to __APP_NAME__ in django templates --- application-templates/django-app/api/templates/main.jinja2 | 2 +- application-templates/django-app/backend/__APP_NAME__/apps.py | 2 +- .../django-app/backend/__APP_NAME__/controllers/__init__.py | 2 +- .../django-app/backend/__APP_NAME__/settings.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-app/api/templates/main.jinja2 index d3177fd78..6d859f736 100644 --- a/application-templates/django-app/api/templates/main.jinja2 +++ b/application-templates/django-app/api/templates/main.jinja2 @@ -20,7 +20,7 @@ apps.populate(settings.INSTALLED_APPS) # migrate the Django models os.system("python manage.py migrate") -from api.controllers import * +from __APP_NAME__.controllers import * app = FastAPI( {% if info %} diff --git a/application-templates/django-app/backend/__APP_NAME__/apps.py b/application-templates/django-app/backend/__APP_NAME__/apps.py index 66656fd29..456a2f40d 100644 --- a/application-templates/django-app/backend/__APP_NAME__/apps.py +++ b/application-templates/django-app/backend/__APP_NAME__/apps.py @@ -3,4 +3,4 @@ class ApiConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' - name = 'api' + name = '__APP_NAME__' diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py b/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py index 1344f5cc9..71fac3207 100644 --- a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py +++ b/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py @@ -1 +1 @@ -import api.controllers.test as test_controller +import __APP_NAME__.controllers.test as test_controller diff --git a/application-templates/django-app/backend/__APP_NAME__/settings.py b/application-templates/django-app/backend/__APP_NAME__/settings.py index 9b3c6d221..cebc607c6 100644 --- a/application-templates/django-app/backend/__APP_NAME__/settings.py +++ b/application-templates/django-app/backend/__APP_NAME__/settings.py @@ -130,8 +130,8 @@ # add the local apps INSTALLED_APPS += [ - "api", - "__APP_NAME__" + "__APP_NAME__", + "ch_django" ] # override django admin base template with a local template From 21c6167c80cc17c0bc8214da9f108a116ef77598 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 16 Sep 2024 09:57:41 +0100 Subject: [PATCH 05/21] CH-59 Rename ch_django directory to django_baseapp directory in the django template --- .../django-app/backend/__APP_NAME__/migrations/0001_initial.py | 2 +- .../django-app/backend/__APP_NAME__/settings.py | 2 +- .../backend/{ch_django => django_baseapp}/__init__.py | 0 .../django-app/backend/{ch_django => django_baseapp}/asgi.py | 0 .../{ch_django => django_baseapp}/migrations/0001_initial.py | 0 .../{ch_django => django_baseapp}/migrations/__init__.py | 0 .../django-app/backend/{ch_django => django_baseapp}/models.py | 0 .../backend/{ch_django => django_baseapp}/static/www/index.html | 0 .../templates/__APP_NAME__/index.html | 0 .../templates/__APP_NAME__/swagger-ui.html | 0 .../django-app/backend/{ch_django => django_baseapp}/urls.py | 0 .../django-app/backend/{ch_django => django_baseapp}/views.py | 0 .../django-app/backend/{ch_django => django_baseapp}/wsgi.py | 0 13 files changed, 2 insertions(+), 2 deletions(-) rename application-templates/django-app/backend/{ch_django => django_baseapp}/__init__.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/asgi.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/migrations/0001_initial.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/migrations/__init__.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/models.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/static/www/index.html (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/templates/__APP_NAME__/index.html (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/templates/__APP_NAME__/swagger-ui.html (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/urls.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/views.py (100%) rename application-templates/django-app/backend/{ch_django => django_baseapp}/wsgi.py (100%) diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py index 34c1f836d..e3f84b27d 100644 --- a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py +++ b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py @@ -4,7 +4,7 @@ class Migration(migrations.Migration): dependencies = [ - ("ch_django", "0001_initial"), + ("django_baseapp", "0001_initial"), ] operations = [] \ No newline at end of file diff --git a/application-templates/django-app/backend/__APP_NAME__/settings.py b/application-templates/django-app/backend/__APP_NAME__/settings.py index cebc607c6..28077f391 100644 --- a/application-templates/django-app/backend/__APP_NAME__/settings.py +++ b/application-templates/django-app/backend/__APP_NAME__/settings.py @@ -131,7 +131,7 @@ # add the local apps INSTALLED_APPS += [ "__APP_NAME__", - "ch_django" + "django_baseapp" ] # override django admin base template with a local template diff --git a/application-templates/django-app/backend/ch_django/__init__.py b/application-templates/django-app/backend/django_baseapp/__init__.py similarity index 100% rename from application-templates/django-app/backend/ch_django/__init__.py rename to application-templates/django-app/backend/django_baseapp/__init__.py diff --git a/application-templates/django-app/backend/ch_django/asgi.py b/application-templates/django-app/backend/django_baseapp/asgi.py similarity index 100% rename from application-templates/django-app/backend/ch_django/asgi.py rename to application-templates/django-app/backend/django_baseapp/asgi.py diff --git a/application-templates/django-app/backend/ch_django/migrations/0001_initial.py b/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/ch_django/migrations/0001_initial.py rename to application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/ch_django/migrations/__init__.py b/application-templates/django-app/backend/django_baseapp/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/ch_django/migrations/__init__.py rename to application-templates/django-app/backend/django_baseapp/migrations/__init__.py diff --git a/application-templates/django-app/backend/ch_django/models.py b/application-templates/django-app/backend/django_baseapp/models.py similarity index 100% rename from application-templates/django-app/backend/ch_django/models.py rename to application-templates/django-app/backend/django_baseapp/models.py diff --git a/application-templates/django-app/backend/ch_django/static/www/index.html b/application-templates/django-app/backend/django_baseapp/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/ch_django/static/www/index.html rename to application-templates/django-app/backend/django_baseapp/static/www/index.html diff --git a/application-templates/django-app/backend/ch_django/templates/__APP_NAME__/index.html b/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/ch_django/templates/__APP_NAME__/index.html rename to application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/ch_django/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/ch_django/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/ch_django/urls.py b/application-templates/django-app/backend/django_baseapp/urls.py similarity index 100% rename from application-templates/django-app/backend/ch_django/urls.py rename to application-templates/django-app/backend/django_baseapp/urls.py diff --git a/application-templates/django-app/backend/ch_django/views.py b/application-templates/django-app/backend/django_baseapp/views.py similarity index 100% rename from application-templates/django-app/backend/ch_django/views.py rename to application-templates/django-app/backend/django_baseapp/views.py diff --git a/application-templates/django-app/backend/ch_django/wsgi.py b/application-templates/django-app/backend/django_baseapp/wsgi.py similarity index 100% rename from application-templates/django-app/backend/ch_django/wsgi.py rename to application-templates/django-app/backend/django_baseapp/wsgi.py From ae198ee6025d00c673250fd7f1efecf754a08a87 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 16 Sep 2024 16:22:43 +0100 Subject: [PATCH 06/21] CH-59 Fix linting issues --- .../django-app/backend/__APP_NAME__/migrations/0001_initial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py index e3f84b27d..8120bb654 100644 --- a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py +++ b/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py @@ -7,4 +7,4 @@ class Migration(migrations.Migration): ("django_baseapp", "0001_initial"), ] - operations = [] \ No newline at end of file + operations = [] From 6201a14fefebba7aa3766790f747345457943a03 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 24 Sep 2024 10:49:08 +0100 Subject: [PATCH 07/21] CH-59 Move settings back to the django_baseapp in the django-app template --- .../backend/{__APP_NAME__ => django_baseapp}/settings.py | 4 ++-- application-templates/django-app/backend/manage.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename application-templates/django-app/backend/{__APP_NAME__ => django_baseapp}/settings.py (98%) diff --git a/application-templates/django-app/backend/__APP_NAME__/settings.py b/application-templates/django-app/backend/django_baseapp/settings.py similarity index 98% rename from application-templates/django-app/backend/__APP_NAME__/settings.py rename to application-templates/django-app/backend/django_baseapp/settings.py index 28077f391..78ba0cbc9 100644 --- a/application-templates/django-app/backend/__APP_NAME__/settings.py +++ b/application-templates/django-app/backend/django_baseapp/settings.py @@ -53,7 +53,7 @@ ] -ROOT_URLCONF = "__APP_NAME__.urls" +ROOT_URLCONF = "django_baseapp.urls" TEMPLATES = [ { @@ -71,7 +71,7 @@ }, ] -WSGI_APPLICATION = "__APP_NAME__.wsgi.application" +WSGI_APPLICATION = "django_baseapp.wsgi.application" # Password validation diff --git a/application-templates/django-app/backend/manage.py b/application-templates/django-app/backend/manage.py index 0412a721a..d106b9543 100644 --- a/application-templates/django-app/backend/manage.py +++ b/application-templates/django-app/backend/manage.py @@ -6,7 +6,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: From 21a21ca77ccde903f19e409e7a3919cf710f66dc Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 24 Sep 2024 11:39:35 +0100 Subject: [PATCH 08/21] CH-59 Add back in the removed shutil import --- tools/deployment-cli-tools/harness-application | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 551d0ae1d..9b5388585 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -3,6 +3,7 @@ import sys import os import re +import shutil import tempfile import subprocess import logging From 1a604bdb3041b7cfd77c934c2f4bc1bd6b1fb928 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 24 Sep 2024 16:23:46 +0100 Subject: [PATCH 09/21] CH-59 Update django settings location --- application-templates/django-app/api/templates/main.jinja2 | 2 +- application-templates/django-app/backend/django_baseapp/wsgi.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-app/api/templates/main.jinja2 index 576f8aee9..5c88b48f1 100644 --- a/application-templates/django-app/api/templates/main.jinja2 +++ b/application-templates/django-app/api/templates/main.jinja2 @@ -14,7 +14,7 @@ from fastapi.staticfiles import StaticFiles {{imports | replace(".","openapi.")}} -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") apps.populate(settings.INSTALLED_APPS) # migrate the Django models diff --git a/application-templates/django-app/backend/django_baseapp/wsgi.py b/application-templates/django-app/backend/django_baseapp/wsgi.py index 678932f0f..e2bea2b41 100644 --- a/application-templates/django-app/backend/django_baseapp/wsgi.py +++ b/application-templates/django-app/backend/django_baseapp/wsgi.py @@ -11,7 +11,7 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") application = get_wsgi_application() From fcf609c6887637890449f1b150d53c3a1f138fa3 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 24 Sep 2024 16:27:13 +0100 Subject: [PATCH 10/21] CH-59 Update views import to import from django_baseapp --- application-templates/django-app/backend/django_baseapp/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application-templates/django-app/backend/django_baseapp/urls.py b/application-templates/django-app/backend/django_baseapp/urls.py index ce1b9629e..ecfee23ee 100644 --- a/application-templates/django-app/backend/django_baseapp/urls.py +++ b/application-templates/django-app/backend/django_baseapp/urls.py @@ -19,7 +19,7 @@ from django.contrib import admin from django.urls import path, re_path -from __APP_NAME__.views import index +from django_baseapp.views import index urlpatterns = [path("admin/", admin.site.urls)] From 37a628e421c35355627ffc3a33bed88ed42216d7 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 12:13:06 +0100 Subject: [PATCH 11/21] CH-149 Refactor harness-application script for better maintainability --- .../deployment-cli-tools/harness-application | 166 ++++++++++++------ tools/deployment-cli-tools/requirements.txt | 3 +- tools/deployment-cli-tools/setup.py | 4 +- 3 files changed, 114 insertions(+), 59 deletions(-) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 9b5388585..a2ffdcea5 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -7,6 +7,7 @@ import shutil import tempfile import subprocess import logging +import argparse from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH @@ -14,19 +15,70 @@ from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLI from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ to_python_module, copymergedir +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' -if __name__ == "__main__": - import argparse - parser = argparse.ArgumentParser( - description='Creates a new Application.') +class TemplateType(StrEnum): + BASE = 'base' + FLASK_SERVER = 'flask-server' + WEBAPP = 'webapp' + DB_POSTGRES = 'db-postgres' + DB_NEO4J = 'db-neo4j' + DB_MONGO = 'db-mongo' + DJANGO_APP = 'django-app' + SERVER = 'server' + + +def main(): + app_name, templates = get_command_line_arguments() + + app_path = os.path.join(APPLICATIONS_SRC_PATH, app_name) + os.makedirs(app_path, exist_ok=True) + + if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: + templates = [TemplateType.BASE, TemplateType.WEBAPP] + templates + + if TemplateType.WEBAPP in templates: + handle_webapp_template(app_name, app_path) + + if TemplateType.SERVER in templates: + handle_server_template(app_path) + + for template_name in templates: + merge_template_directories(template_name, app_path) + + if TemplateType.FLASK_SERVER in templates: + handle_flask_server_template(app_path) + + replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(app_name)) + + if TemplateType.DJANGO_APP in templates: + handle_django_app_template(app_name, app_path) + + replaceindir(app_path, PLACEHOLDER, app_name) + + if TemplateType.WEBAPP in templates: + handle_webapp_template_cleanup(app_path) + + +def get_command_line_arguments() -> tuple[str, list[str]]: + parser = argparse.ArgumentParser(description='Creates a new Application.') + parser.add_argument('name', metavar='name', type=str, help='Application name') - parser.add_argument('-t', '--template', dest='templates', action="append", default=['base', ], + parser.add_argument('-t', '--template', + dest='templates', + action="append", + default=[TemplateType.BASE], + type=str, help="""Add a template name. Available templates: @@ -55,58 +107,60 @@ if __name__ == "__main__": print("Invalid regex") exit(1) - app_path = os.path.join(APPLICATIONS_SRC_PATH, args.name) - os.makedirs(app_path, exist_ok=True) - templates = args.templates - - if "django-app" in args.templates and "webapp" not in templates: - templates = ["base", "webapp"] + templates - - if 'webapp' in templates: - if os.path.exists(os.path.join(app_path, 'frontend')): - shutil.rmtree(os.path.join(app_path, 'frontend')) - cmd = ["yarn", "create", "vite", args.name, "--template", "react-ts"] - logging.info(f"Running command: {' '.join(cmd)}") - subprocess.run(cmd, cwd=app_path) - shutil.move(os.path.join(app_path, args.name), os.path.join(app_path, 'frontend')) - generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) - - if 'server' in templates: - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) + return args.name, args.templates - for template_name in templates: - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) - if "flask-server" in templates: +def handle_webapp_template(app_name: str, app_path: str) -> None: + if os.path.exists(os.path.join(app_path, 'frontend')): + shutil.rmtree(os.path.join(app_path, 'frontend')) + cmd = ["yarn", "create", "vite", app_name, "--template", "react-ts"] + logging.info(f"Running command: {' '.join(cmd)}") + subprocess.run(cmd, cwd=app_path) + shutil.move(os.path.join(app_path, app_name), os.path.join(app_path, 'frontend')) + generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) + + +def handle_webapp_template_cleanup(app_path: str) -> None: + try: + os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) + except FileNotFoundError: + # backend dockerfile not found, continue + pass + + +def handle_server_template(app_path: str) -> None: + with tempfile.TemporaryDirectory() as tmp_dirname: + copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, TemplateType.SERVER), tmp_dirname) + merge_configuration_directories(app_path, tmp_dirname) + generate_server(app_path, tmp_dirname) + + +def handle_flask_server_template(app_path: str) -> None: generate_server(app_path) - replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(args.name)) - - if "django-app" in templates: - replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(args.name)) - generate_fastapi_server(app_path) - replace_in_file( - os.path.join(app_path, 'deploy/values.yaml'), - f"{PLACEHOLDER}:{PLACEHOLDER}", - f"{to_python_module(args.name)}:{to_python_module(args.name)}" - ) - try: - os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) - except FileNotFoundError: - # backend dockerfile not found, continue - pass - - replaceindir(app_path, PLACEHOLDER, args.name) - - if 'webapp' in templates: - try: - os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) - except FileNotFoundError: - # backend dockerfile not found, continue - pass + +def handle_django_app_template(app_name: str, app_path: str) -> None: + replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(app_name)) + generate_fastapi_server(app_path) + replace_in_file( + os.path.join(app_path, 'deploy/values.yaml'), + f"{PLACEHOLDER}:{PLACEHOLDER}", + f"{to_python_module(app_name)}:{to_python_module(app_name)}" + ) + replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) + try: + os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) + except FileNotFoundError: + # backend dockerfile not found, continue + pass + + +def merge_template_directories(template_name: str, app_path: str) -> None: + for base_path in (CH_ROOT, os.getcwd()): + template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) + if os.path.exists(template_path): + merge_configuration_directories(template_path, app_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/deployment-cli-tools/requirements.txt b/tools/deployment-cli-tools/requirements.txt index 622565b9a..266ae27f3 100644 --- a/tools/deployment-cli-tools/requirements.txt +++ b/tools/deployment-cli-tools/requirements.txt @@ -4,4 +4,5 @@ ruamel.yaml oyaml cloudharness_model cloudharness_utils -dirhash \ No newline at end of file +dirhash +StrEnum ; python_version < '3.11' \ No newline at end of file diff --git a/tools/deployment-cli-tools/setup.py b/tools/deployment-cli-tools/setup.py index b01b2ae24..280cd716b 100644 --- a/tools/deployment-cli-tools/setup.py +++ b/tools/deployment-cli-tools/setup.py @@ -28,10 +28,10 @@ 'cloudharness_model', 'cloudharness_utils', 'fastapi-code-generator', - 'dirhash' + 'dirhash', + "StrEnum ; python_version < '3.11'", ] - setup( name=NAME, version=VERSION, From c8ce9709d46116e6757922cb9db539b6a96761a8 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 12:13:43 +0100 Subject: [PATCH 12/21] CH-149 Add dev setup script for django applications --- application-templates/django-app/dev-setup.sh | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 application-templates/django-app/dev-setup.sh diff --git a/application-templates/django-app/dev-setup.sh b/application-templates/django-app/dev-setup.sh new file mode 100644 index 000000000..7bdd82505 --- /dev/null +++ b/application-templates/django-app/dev-setup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$(pwd) +CH_DIRECTORY="../../cloud-harness" +INSTALL_PYTEST=false +CURRENT_DIRECTORY="$(pwd)" +APP_NAME="__APP_NAME__" + +pip_upgrade_error() { + echo "Unable to upgrade pip" + exit 1 +} + +install_error () { + echo "Unable to install $1" 1>&2 + exit 1 +} + +while getopts ch_directory:pytest arg; +do + case "$arg" in + ch_directory) CH_DIRECTORY=${OPTARG};; + pytest) INSTALL_PYTEST=true;; + esac +done + +pip install --upgrade pip || pip_upgrade_error + +# Install pip dependencies from cloudharness-base-debian image + +if $INSTALL_PYTEST; then + pip install pytest || install_error pytest +fi + +pip install -r "$CH_DIRECTORY/libraries/models/requirements.txt" || install_error "models requirements" +pip install -r "$CH_DIRECTORY/libraries/cloudharness-common/requirements.txt" || install_error "cloudharness-common requirements" +pip install -r "$CH_DIRECTORY/libraries/client/cloudharness_cli/requirements.txt" || install_error "cloudharness_cli requirements" + +pip install -e "$CH_DIRECTORY/libraries/models" || install_error models +pip install -e "$CH_DIRECTORY/libraries/cloudharness-common" || install_error cloudharness-common +pip install -e "$CH_DIRECTORY/libraries/client/cloudharness_cli" || install_error cloudharness_cli + +# Install pip dependencies from cloudharness-django image + +pip install -r "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/fastapi/requirements.txt" || install_error "cloudharness-django fastapi requirements" +pip install -e "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django" || install_error cloudharness-django + +# Install application + +pip install -r "$CURRENT_DIRECTORY/backend/requirements.txt" || install_error "$APP_NAME dependencies" +pip install -e "$CURRENT_DIRECTORY/backend" || install_error "$APP_NAME" \ No newline at end of file From 63a4e1862533bdc39bcd6cbbed6c164e9083d067 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 12:14:42 +0100 Subject: [PATCH 13/21] CH-149 Change cloudharness-django depdencies from fixed versions to min versions --- .../libraries/cloudharness-django/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg index d298dece8..5ca2ead8d 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/setup.cfg @@ -31,6 +31,6 @@ python_requires = >=3.6 install_requires = Django>=4.0.7 django-admin-extra-buttons>=1.4.2 - psycopg2-binary==2.9.3 - Pillow==9.2.0 + psycopg2-binary>=2.9.3 + Pillow>=9.2.0 python-keycloak From 795b7d4e56d359a48346bd657c6ec41b8fd6cefa Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 15:15:48 +0100 Subject: [PATCH 14/21] CH-149 Add debug configuration to vscode launch settings when creating a django app --- .../vscode-django-app-debug-template.json | 24 +++++++++++++++++ .../ch_cli_tools/utils.py | 17 ++++++++++++ .../deployment-cli-tools/harness-application | 26 ++++++++++++++++++- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 deployment-configuration/vscode-django-app-debug-template.json diff --git a/deployment-configuration/vscode-django-app-debug-template.json b/deployment-configuration/vscode-django-app-debug-template.json new file mode 100644 index 000000000..f049d5e53 --- /dev/null +++ b/deployment-configuration/vscode-django-app-debug-template.json @@ -0,0 +1,24 @@ +{ + "args": [ + "--host", + "0.0.0.0", + "--port", + "8000", + "main:app" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/applications/__APP_NAME__/backend", + "env": { + "ACCOUNTS_ADMIN_PASSWORD": "metacell", + "ACCOUNTS_ADMIN_USERNAME": "admin", + "CH_CURRENT_APP_NAME": "__APP_NAME__", + "CH_VALUES_PATH": "${workspaceFolder}/deployment/helm/values.yaml", + "DJANGO_SETTINGS_MODULE": "django_baseapp.settings", + "KUBERNETES_SERVICE_HOST": "ssdds" + }, + "justMyCode": false, + "module": "uvicorn", + "name": "__APP_NAME__ backend", + "request": "launch", + "type": "debugpy" + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 7a00626d9..17e5c1548 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -2,6 +2,7 @@ import socket import glob import subprocess +from typing import Any import requests import os from functools import cache @@ -188,6 +189,22 @@ def replace_in_file(src_file, source, replace): pass +def replace_in_dict(src_dict: dict, source: str, replacement: str) -> dict: + def replace_value(value: Any) -> Any: + if isinstance(value, str): + return value.replace(source, replacement) + if isinstance(value, list): + return [replace_value(item) for item in value] + if isinstance(value, dict): + return replace_in_dict(value, source, replacement) + return value + + return { + key: replace_value(value) + for key, value in src_dict.items() + } + + def copymergedir(root_src_dir, root_dst_dir): """ Does copy and merge (shutil.copytree requires that the destination does not exist) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index a2ffdcea5..36b3e0403 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -1,5 +1,7 @@ #!/usr/bin/env python +import json +import pathlib import sys import os import re @@ -13,7 +15,7 @@ from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ - to_python_module, copymergedir + to_python_module, copymergedir, get_json_template, replace_in_dict try: from enum import StrEnum @@ -148,6 +150,7 @@ def handle_django_app_template(app_name: str, app_path: str) -> None: f"{to_python_module(app_name)}:{to_python_module(app_name)}" ) replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) + create_django_app_vscode_debug_configuration(app_name) try: os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) except FileNotFoundError: @@ -155,6 +158,27 @@ def handle_django_app_template(app_name: str, app_path: str) -> None: pass +def create_django_app_vscode_debug_configuration(app_name: str): + vscode_launch_path = pathlib.Path('.vscode/launch.json') + configuration_name = f'{app_name} backend' + + launch_config = get_json_template(vscode_launch_path, True) + + launch_config['configurations'] = [ + configuration for configuration in launch_config['configurations'] + if configuration['name'] != configuration_name + ] + + debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = replace_in_dict(debug_config, PLACEHOLDER, app_name) + + launch_config['configurations'].append(debug_config) + + vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) + with vscode_launch_path.open('w') as f: + json.dump(launch_config, f, indent=2, sort_keys=True) + + def merge_template_directories(template_name: str, app_path: str) -> None: for base_path in (CH_ROOT, os.getcwd()): template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) From d4937fbaa7652b16223aa6eb5c8b754a178a47e9 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 16:59:04 +0100 Subject: [PATCH 15/21] CH-67 Make events listener initialization an explicit action instead of implicit on import --- .../django-app/backend/django_baseapp/asgi.py | 4 +++- .../django-app/backend/django_baseapp/wsgi.py | 4 +++- .../cloudharness-django/cloudharness_django/README.md | 4 +++- .../cloudharness_django/services/events.py | 8 ++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/application-templates/django-app/backend/django_baseapp/asgi.py b/application-templates/django-app/backend/django_baseapp/asgi.py index 6d1bf4d96..049df40e6 100644 --- a/application-templates/django-app/backend/django_baseapp/asgi.py +++ b/application-templates/django-app/backend/django_baseapp/asgi.py @@ -21,4 +21,6 @@ init_services() # start the kafka event listener -import cloudharness_django.services.events # noqa E402 +from cloudharness_django.services.events import init_listner # noqa E402 + +init_listner() diff --git a/application-templates/django-app/backend/django_baseapp/wsgi.py b/application-templates/django-app/backend/django_baseapp/wsgi.py index e2bea2b41..e2d6fc099 100644 --- a/application-templates/django-app/backend/django_baseapp/wsgi.py +++ b/application-templates/django-app/backend/django_baseapp/wsgi.py @@ -21,4 +21,6 @@ init_services() # start the kafka event listener -import cloudharness_django.services.events # noqa E402 +from cloudharness_django.services.events import init_listner # noqa E402 + +init_listner() diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md index e6d7074bd..9e4eda591 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/README.md @@ -46,7 +46,9 @@ Quick start init_services() # start the kafka event listener - import cloudharness_django.services.events + from cloudharness_django.services.events import init_listner # noqa E402 + + init_listner() ``` 4. Start the development server and visit http://127.0.0.1:8000/ diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index 55909afe2..1cbd1e933 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -79,8 +79,8 @@ def setup_event_service(self): pass -# start services -if not hasattr(settings, "PROJECT_NAME"): - raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") +def init_listener(): + if not hasattr(settings, "PROJECT_NAME"): + raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") -KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() + KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() \ No newline at end of file From 2f7b47423be86816f1cde5d3a17a703163efd283 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 17:18:12 +0100 Subject: [PATCH 16/21] CH-67 Change listener initialization to use a singleton and the setup_event_service call idempotent --- .../cloudharness_django/services/events.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index 1cbd1e933..c199616a1 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -15,6 +15,7 @@ class KeycloakMessageService: def __init__(self, kafka_group_id): self._topic = "keycloak.fct.admin" self.kafka_group_id = kafka_group_id + self.topics_initialized = False @staticmethod def event_handler(app, event_client, message): @@ -50,6 +51,9 @@ def event_handler(app, event_client, message): raise e def init_topics(self): + if self.topics_initialized: + return + log.info("Starting Kafka consumer threads") try: event_client = EventClient(self._topic) @@ -59,6 +63,7 @@ def init_topics(self): except TopicAlreadyExistsError as e: pass event_client.async_consume(app={}, group_id=self.kafka_group_id, handler=KeycloakMessageService.event_handler) + self.topics_initialized = True except Exception as e: log.error(f"Error creating topic {self._topic}", exc_info=e) @@ -79,8 +84,15 @@ def setup_event_service(self): pass +_message_service_singleton = None + + def init_listener(): if not hasattr(settings, "PROJECT_NAME"): raise KeycloakOIDCNoProjectError("Project name not found, please set PROJECT_NAME in your settings module") - KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() \ No newline at end of file + global _message_service_singleton + if _message_service_singleton is None: + _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() + + _message_service_singleton.setup_event_service() From 2a1824ab6cc02e9c22b7b10d7511ba349e2158f1 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 26 Sep 2024 17:18:53 +0100 Subject: [PATCH 17/21] CH-67 Add background initialization for auth and events to main.py template --- .../django-app/api/templates/main.jinja2 | 8 +++--- .../cloudharness_django/services/__init__.py | 25 +++++++++++++++++++ .../cloudharness_django/services/events.py | 20 +++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-app/api/templates/main.jinja2 index 5c88b48f1..2afff27a6 100644 --- a/application-templates/django-app/api/templates/main.jinja2 +++ b/application-templates/django-app/api/templates/main.jinja2 @@ -66,10 +66,12 @@ async def add_process_time_header(request: Request, call_next): if os.environ.get('KUBERNETES_SERVICE_HOST', None): # init the auth service when running in/for k8s - from cloudharness_django.services import init_services, get_auth_service - init_services() + from cloudharness_django.services import init_services_in_background, get_auth_service + init_services_in_background() + # start the kafka event listener when running in/for k8s - import cloudharness_django.services.events + from cloudharness_django.services.events import init_listener_in_background + init_listener_in_background() async def has_access(): """ diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py index 4efca5168..330a760eb 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/__init__.py @@ -43,3 +43,28 @@ def init_services( admin_role=admin_role) _user_service = UserService(_auth_service) return _auth_service + + +def init_services_in_background( + client_name: str = settings.KC_CLIENT_NAME, + client_roles: List[str] = settings.KC_ALL_ROLES, + privileged_roles: List[str] = settings.KC_PRIVILEGED_ROLES, + admin_role: str = settings.KC_ADMIN_ROLE, + default_user_role: str = settings.KC_DEFAULT_USER_ROLE +): + import threading + import time + from cloudharness import log + + def background_operation(): + services_initialized = False + + while not services_initialized: + try: + init_services(client_name, client_roles, privileged_roles, admin_role, default_user_role) + services_initialized = True + except: + log.exception("Error initializing services. Retrying in 5 seconds...") + time.sleep(5) + + threading.Thread(target=background_operation).start() diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index c199616a1..62d9c7f75 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -96,3 +96,23 @@ def init_listener(): _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() _message_service_singleton.setup_event_service() + + +def init_listener_in_background(): + import threading + import time + from cloudharness import log + + def background_operation(): + listener_initialized = False + + while not listener_initialized: + try: + init_listener() + log.info('User sync events listener started') + listener_initialized = True + except: + log.exception('Error initializing event queue. Retrying in 5 seconds...') + time.sleep(5) + + threading.Thread(target=background_operation).start() From b2918426291f38913c384ccbb389f69c05f0a8a4 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 27 Sep 2024 10:50:55 +0100 Subject: [PATCH 18/21] CH-149 Update command in generate_fastapi_server --- tools/deployment-cli-tools/ch_cli_tools/openapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index e6f232234..4df7b8834 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -33,7 +33,7 @@ def generate_server(app_path, overrides_folder=""): def generate_fastapi_server(app_path): - command = f"cd {app_path}/api && bash -c ./genapi.sh" + command = f"cd {app_path}/api && bash genapi.sh" os.system(command) From 0dfd8c86b845091a9b53e47818c8b65d5d39464b Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 27 Sep 2024 11:24:45 +0100 Subject: [PATCH 19/21] CH-149 Add unit tests for the added utility --- .../deployment-cli-tools/tests/test_utils.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 78e843334..0ce932fee 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -87,3 +87,65 @@ def test_find_dockerfile_paths(): assert len(dockerfiles) == 2 assert next(d for d in dockerfiles if d.endswith("myapp")), "Must find the Dockerfile in the root directory" assert next(d for d in dockerfiles if d.endswith("myapp/tasks/mytask")), "Must find the Dockerfile in the tasks directory" + + +class TestReplaceInDict: + def test_does_not_replace_in_keys(_): + src_dict = { + 'foo': 1, + 'bar': 2, + 'baz': 3, + 'foobar': 4, + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict.keys() == src_dict.keys() + + def test_replaces_in_values(_): + src_dict = { + 'a': 'foo', + 'b': 'bar', + 'c': 'baz', + 'd': 3, + 'e': 'foobar', + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict == { + 'a': 'xxx', + 'b': 'bar', + 'c': 'baz', + 'd': 3, + 'e': 'xxxbar', + } + + def test_replaces_in_values_within_lists(_): + src_dict = { + 'a': ['foo', 'bar', 'baz', 3, 'foobar'], + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict['a'] == ['xxx', 'bar', 'baz', 3, 'xxxbar'] + + + def test_replaces_in_values_within_nested_dict(_): + src_dict = { + 'a': { + 'a': 'foo', + 'b': 'bar', + 'c': 'foobar', + 'e': ['foo', 'bar', 'foobar'], + }, + } + + new_dict = replace_in_dict(src_dict, 'foo', 'xxx') + + assert new_dict['a'] == { + 'a': 'xxx', + 'b': 'bar', + 'c': 'xxxbar', + 'e': ['xxx', 'bar', 'xxxbar'] + } \ No newline at end of file From 09d984259da2aef8ad1a1dde372ab3a49e2f0101 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 27 Sep 2024 11:30:55 +0100 Subject: [PATCH 20/21] CH-59 Fix linting errors --- .../cloudharness-common/cloudharness/middleware/django.py | 1 + tools/deployment-cli-tools/ch_cli_tools/utils.py | 2 +- tools/deployment-cli-tools/harness-application | 6 +++--- tools/deployment-cli-tools/tests/test_utils.py | 5 ++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/cloudharness-common/cloudharness/middleware/django.py b/libraries/cloudharness-common/cloudharness/middleware/django.py index e78837511..f3f6ee760 100644 --- a/libraries/cloudharness-common/cloudharness/middleware/django.py +++ b/libraries/cloudharness-common/cloudharness/middleware/django.py @@ -1,6 +1,7 @@ from cloudharness.middleware import set_authentication_token from django.http.request import HttpRequest + class CloudharnessMiddleware: def __init__(self, get_response): self.get_response = get_response diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 17e5c1548..85fcb5bba 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -198,7 +198,7 @@ def replace_value(value: Any) -> Any: if isinstance(value, dict): return replace_in_dict(value, source, replacement) return value - + return { key: replace_value(value) for key, value in src_dict.items() diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 36b3e0403..d3601f7a5 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -138,7 +138,7 @@ def handle_server_template(app_path: str) -> None: def handle_flask_server_template(app_path: str) -> None: - generate_server(app_path) + generate_server(app_path) def handle_django_app_template(app_name: str, app_path: str) -> None: @@ -177,7 +177,7 @@ def create_django_app_vscode_debug_configuration(app_name: str): vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) with vscode_launch_path.open('w') as f: json.dump(launch_config, f, indent=2, sort_keys=True) - + def merge_template_directories(template_name: str, app_path: str) -> None: for base_path in (CH_ROOT, os.getcwd()): @@ -187,4 +187,4 @@ def merge_template_directories(template_name: str, app_path: str) -> None: if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/deployment-cli-tools/tests/test_utils.py b/tools/deployment-cli-tools/tests/test_utils.py index 0ce932fee..0c4060f36 100644 --- a/tools/deployment-cli-tools/tests/test_utils.py +++ b/tools/deployment-cli-tools/tests/test_utils.py @@ -101,7 +101,7 @@ def test_does_not_replace_in_keys(_): new_dict = replace_in_dict(src_dict, 'foo', 'xxx') assert new_dict.keys() == src_dict.keys() - + def test_replaces_in_values(_): src_dict = { 'a': 'foo', @@ -130,7 +130,6 @@ def test_replaces_in_values_within_lists(_): assert new_dict['a'] == ['xxx', 'bar', 'baz', 3, 'xxxbar'] - def test_replaces_in_values_within_nested_dict(_): src_dict = { 'a': { @@ -148,4 +147,4 @@ def test_replaces_in_values_within_nested_dict(_): 'b': 'bar', 'c': 'xxxbar', 'e': ['xxx', 'bar', 'xxxbar'] - } \ No newline at end of file + } From e11608f2d1c1fc3a2d27715d635a8afc39313470 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 27 Sep 2024 13:35:41 +0100 Subject: [PATCH 21/21] CH-67 Fix incorrect assignment of singleton --- .../cloudharness-django/cloudharness_django/services/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py index 62d9c7f75..170b41ac0 100644 --- a/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py +++ b/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/events.py @@ -93,7 +93,7 @@ def init_listener(): global _message_service_singleton if _message_service_singleton is None: - _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME).setup_event_service() + _message_service_singleton = KeycloakMessageService(settings.PROJECT_NAME) _message_service_singleton.setup_event_service()