From d671a285ea1f288d352ba2cd4f74ba9eea278503 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Thu, 31 Oct 2024 11:42:10 +0000 Subject: [PATCH 01/15] CH-157 Move harness-application builder steps to a more structured pipeline approach --- .../ch_cli_tools/application_builders.py | 245 ++++++++++++++++++ .../deployment-cli-tools/harness-application | 108 +------- 2 files changed, 250 insertions(+), 103 deletions(-) create mode 100644 tools/deployment-cli-tools/ch_cli_tools/application_builders.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py new file mode 100644 index 00000000..3ce72785 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -0,0 +1,245 @@ +import abc +import json +import logging +import pathlib +import shutil +import subprocess +import tempfile +from .common_types import TemplateType +from .openapi import generate_fastapi_server, generate_server, generate_ts_client +from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, to_python_module +from . import CH_ROOT +from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH + + +class ApplicationBuilder(abc.ABC): + APP_NAME_PLACEHOLDER = '__APP_NAME__' + + def __init__(self, app_name: str, app_path: pathlib.Path): + self.app_name = app_name + self.app_path = app_path + + @abc.abstractmethod + def handles(self, templates: list[str]) -> bool: + pass + + @abc.abstractmethod + def handle_pre_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_merge(self) -> None: + pass + + @abc.abstractmethod + def handle_post_merge(self) -> None: + pass + + def run_command(self, *command: str, cwd: pathlib.Path = None) -> None: + if not cwd: + cwd = self.app_path + + logging.info(f'Running command: {" ".join(map(str, command))}') + subprocess.run(command, cwd=cwd) + + def merge_template_directories(self, template_name: str) -> None: + for base_path in (self.ch_root, pathlib.Path.cwd()): + template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name + if template_path.exists(): + merge_configuration_directories(template_path, self.app_path) + + @property + def frontend_path(self): + return self.app_path / 'frontend' + + @property + def backend_path(self): + return self.app_path / 'backend' + + @property + def api_path(self): + return self.app_path / 'api' + + @property + def ch_root(self): + return pathlib.Path(CH_ROOT) + + @property + def app_template_path(self): + return self.ch_root / APPLICATION_TEMPLATE_PATH + + +class WebAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.WEBAPP in templates + + def handle_pre_merge(self): + if self.frontend_path.exists(): + shutil.rmtree(self.frontend_path) + + self.create_vite_skaffold(self.frontend_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.WEBAPP) + + def handle_post_merge(self): + backend_dockerfile_path = self.backend_path / 'Dockerfile' + backend_dockerfile_path.unlink(missing_ok=True) + + self.install_frontend_dependencies() + generate_ts_client(self.api_path / 'openapi.yaml') + + def create_vite_skaffold(self, frontend_path: pathlib.Path) -> None: + self.run_command( + 'yarn', 'create', 'vite', self.app_name, + '--template', 'react-ts', + ) + shutil.move(self.app_path / self.app_name, frontend_path) + + def install_frontend_dependencies(self) -> None: + self.run_command('yarn', 'install', cwd=self.frontend_path) + + +class ServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.SERVER in templates + + def handle_pre_merge(self): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = self.app_template_path / TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(self.app_path, tmp_path) + generate_server(self.app_name, tmp_path) + + def handle_merge(self): + self.merge_template_directories(TemplateType.SERVER) + + def handle_post_merge(self): + pass + + +class FlaskServerAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.FLASK_SERVER in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + self.merge_template_directories(TemplateType.FLASK_SERVER) + + def handle_post_merge(self): + generate_server(self.app_path) + + +class DjangoAppBuilder(ApplicationBuilder): + def handles(self, templates): + return TemplateType.DJANGO_APP in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + self.merge_template_directories(TemplateType.DJANGO_APP) + + def handle_post_merge(self): + replace_in_file( + self.api_path / 'templates' / 'main.jinja2', + self.APP_NAME_PLACEHOLDER, + self.python_app_name, + ) + replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + generate_fastapi_server(self.app_path) + + replace_in_file( + self.app_path / 'deploy' / 'values.yaml', + f'{self.APP_NAME_PLACEHOLDER}:{self.APP_NAME_PLACEHOLDER}', + f'{self.python_app_name}:{self.python_app_name}', + ) + replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + + self.create_django_app_vscode_debug_configuration() + + (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) + + @property + def python_app_name(self): + return to_python_module(self.app_name) + + def create_django_app_vscode_debug_configuration(self): + vscode_launch_path = pathlib.Path('.vscode/launch.json') + configuration_name = f'{self.app_name} backend' + + launch_config = get_json_template(vscode_launch_path, True) + + launch_config['configurations'] = [ + configuration for configuration in launch_config['configurations'] + if configuration['name'] != configuration_name + ] + + debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = replace_in_dict(debug_config, self.APP_NAME_PLACEHOLDER, self.app_name) + + launch_config['configurations'].append(debug_config) + + vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) + with vscode_launch_path.open('w') as f: + json.dump(launch_config, f, indent=2, sort_keys=True) + + +class AppBuilderPipeline(ApplicationBuilder): + def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): + super().__init__(app_name, app_path) + self.templates = templates + self.app_builders: dict[str, ApplicationBuilder] = { + TemplateType.WEBAPP: WebAppBuilder(app_name, app_path), + TemplateType.SERVER: ServerAppBuilder(app_name, app_path), + TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), + TemplateType.DJANGO_APP: DjangoAppBuilder(app_name, app_path), + } + + def handles(self, templates): + return templates == self.templates + + def handle_pre_merge(self): + pre_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_APP, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in pre_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_pre_merge() + + def handle_merge(self): + for template in self.templates: + run_merge = ( + app_builder.handle_merge + if (app_builder := self.app_builders.get(template, None)) + else lambda: self.merge_template_directories(template) + ) + run_merge() + + def handle_post_merge(self): + post_merge_template_order = [ + TemplateType.FLASK_SERVER, + TemplateType.DJANGO_APP, + TemplateType.WEBAPP, + TemplateType.SERVER, + ] + + app_builders = [ + self.app_builders[template] for template in post_merge_template_order + if self.app_builders[template].handles(self.templates) + ] + + for app_builder in app_builders: + app_builder.handle_post_merge() diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index c79fee3f..5cca3d75 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -17,6 +17,7 @@ from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLI from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +from ch_cli_tools.application_builders import AppBuilderPipeline # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") @@ -32,27 +33,15 @@ def main() -> None: templates = normalize_templates(templates) - if TemplateType.WEBAPP in templates: - handle_webapp_template(app_name, app_path) + pipeline = AppBuilderPipeline(app_name, app_path, templates) - if TemplateType.SERVER in templates: - handle_server_template(app_path) - - for template_name in templates: - merge_template_directories(template_name, app_path) - - if TemplateType.FLASK_SERVER in templates: - handle_flask_server_template(app_path) + pipeline.handle_pre_merge() + pipeline.handle_merge() replace_in_file(app_path / 'api' / 'config.json', PLACEHOLDER, to_python_module(app_name)) - - if TemplateType.DJANGO_APP in templates: - handle_django_app_template(app_name, app_path) - replaceindir(app_path, PLACEHOLDER, app_name) - if TemplateType.WEBAPP in templates: - handle_webapp_template_post_merge(app_path) + pipeline.handle_post_merge() create_manifest_file(app_path, app_name, templates) call_harness_generate(app_path, app_name) @@ -114,86 +103,6 @@ def normalize_templates(templates: list[str]) -> list[str]: return normalized_templates -def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: - frontend_path = app_path / 'frontend' - - if frontend_path.exists(): - shutil.rmtree(frontend_path) - - create_vite_skaffold(app_name, app_path, frontend_path) - - -def create_vite_skaffold(app_name: str, app_path: pathlib.Path, frontend_path: pathlib.Path) -> None: - command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] - logging.info(f'Running command: {" ".join(command)}') - subprocess.run(command, cwd=app_path) - shutil.move(app_path / app_name, frontend_path) - - -def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: - command = ['yarn', 'install'] - logging.info(f'Running command: {" ".join(command)}') - subprocess.run(command, cwd=frontend_path) - - -def handle_webapp_template_post_merge(app_path: pathlib.Path) -> None: - backend_dockerfile_path = app_path / 'backend' / 'Dockerfile' - backend_dockerfile_path.unlink(missing_ok=True) - - install_frontend_dependencies(app_path / 'frontend') - generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') - - -def handle_server_template(app_path: pathlib.Path) -> None: - with tempfile.TemporaryDirectory() as tmp_dirname: - tmp_path = pathlib.Path(tmp_dirname) - server_template_path = pathlib.Path(CH_ROOT) / APPLICATION_TEMPLATE_PATH / TemplateType.SERVER - - copymergedir(server_template_path, tmp_path) - merge_configuration_directories(app_path, tmp_path) - generate_server(app_path, tmp_path) - - -def handle_flask_server_template(app_path: pathlib.Path) -> None: - generate_server(app_path) - - -def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: - python_app_name = to_python_module(app_name) - - api_path = app_path / 'api' - replace_in_file(api_path / 'templates' / 'main.jinja2', PLACEHOLDER, python_app_name) - replace_in_file(api_path / 'genapi.sh', PLACEHOLDER, app_name) - generate_fastapi_server(app_path) - - replace_in_file(app_path / 'deploy' / 'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') - replace_in_file(app_path / 'dev-setup.sh', PLACEHOLDER, app_name) - create_django_app_vscode_debug_configuration(app_name) - - (app_path / 'backend' / '__APP_NAME__' / '__main__.py').unlink(missing_ok=True) - - -def create_django_app_vscode_debug_configuration(app_name: str): - vscode_launch_path = pathlib.Path('.vscode/launch.json') - configuration_name = f'{app_name} backend' - - launch_config = get_json_template(vscode_launch_path, True) - - launch_config['configurations'] = [ - configuration for configuration in launch_config['configurations'] - if configuration['name'] != configuration_name - ] - - debug_config = get_json_template('vscode-django-app-debug-template.json', True) - debug_config = replace_in_dict(debug_config, PLACEHOLDER, app_name) - - launch_config['configurations'].append(debug_config) - - vscode_launch_path.parent.mkdir(parents=True, exist_ok=True) - with vscode_launch_path.open('w') as f: - json.dump(launch_config, f, indent=2, sort_keys=True) - - def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: manifest_file = app_path / '.ch-manifest' manifest = CloudHarnessManifest( @@ -214,12 +123,5 @@ def call_harness_generate(app_path: pathlib.Path, app_name: str): subprocess.run(command) -def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: - for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): - template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name - if template_path.exists(): - merge_configuration_directories(template_path, app_path) - - if __name__ == "__main__": main() From ad10cd86164af72aefa185eb47e3f063a8f6596a Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 12:54:02 +0000 Subject: [PATCH 02/15] CH-157 Rename template from django-app to django-fastapi --- .../ch_cli_tools/application_builders.py | 14 ++++++------- .../ch_cli_tools/common_types.py | 19 ++++++++++++++++-- .../deployment-cli-tools/harness-application | 20 +++++++++---------- tools/deployment-cli-tools/harness-generate | 8 +++++--- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 3ce72785..0f5bca0e 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -134,15 +134,15 @@ def handle_post_merge(self): generate_server(self.app_path) -class DjangoAppBuilder(ApplicationBuilder): +class DjangoFastApiBuilder(ApplicationBuilder): def handles(self, templates): - return TemplateType.DJANGO_APP in templates - + return TemplateType.DJANGO_FASTAPI in templates + def handle_pre_merge(self): pass def handle_merge(self): - self.merge_template_directories(TemplateType.DJANGO_APP) + self.merge_template_directories(TemplateType.DJANGO_FASTAPI) def handle_post_merge(self): replace_in_file( @@ -197,7 +197,7 @@ def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): TemplateType.WEBAPP: WebAppBuilder(app_name, app_path), TemplateType.SERVER: ServerAppBuilder(app_name, app_path), TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), - TemplateType.DJANGO_APP: DjangoAppBuilder(app_name, app_path), + TemplateType.DJANGO_FASTAPI: DjangoFastApiBuilder(app_name, app_path), } def handles(self, templates): @@ -206,7 +206,7 @@ def handles(self, templates): def handle_pre_merge(self): pre_merge_template_order = [ TemplateType.FLASK_SERVER, - TemplateType.DJANGO_APP, + TemplateType.DJANGO_FASTAPI, TemplateType.WEBAPP, TemplateType.SERVER, ] @@ -231,7 +231,7 @@ def handle_merge(self): def handle_post_merge(self): post_merge_template_order = [ TemplateType.FLASK_SERVER, - TemplateType.DJANGO_APP, + TemplateType.DJANGO_FASTAPI, TemplateType.WEBAPP, TemplateType.SERVER, ] diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 63f1893d..99b9d41c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -1,3 +1,4 @@ +import copy from dataclasses import dataclass from typing import Union @@ -15,7 +16,7 @@ class TemplateType(StrEnum): DB_POSTGRES = 'db-postgres' DB_NEO4J = 'db-neo4j' DB_MONGO = 'db-mongo' - DJANGO_APP = 'django-app' + DJANGO_FASTAPI = 'django-fastapi' SERVER = 'server' @classmethod @@ -26,9 +27,9 @@ def database_templates(cls): @dataclass class CloudHarnessManifest(): app_name: str - version: str inferred: bool templates: list[str] + version: str = '2' @classmethod def from_dict(cls, data: dict) -> 'CloudHarnessManifest': @@ -46,3 +47,17 @@ def to_dict(self) -> dict: 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], } + + @classmethod + def migrate(cls, data: dict) -> tuple[dict, bool]: + data_copy = copy.deepcopy(data) + update_manifest = False + + if data_copy['version'] < '2': + update_manifest = True + data_copy['templates'] = [ + template if template != 'django-app' else 'django-fastapi' + for template in data_copy['templates'] + ] + + return data_copy, update_manifest \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 5cca3d75..2a65fe37 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -65,7 +65,7 @@ def get_command_line_arguments() -> tuple[str, list[str]]: - db-postgres - db-neo4j - db-mongo - - django-app (fastapi django backend based on openapi) + - django-fastapi (fastapi django backend based on openapi) """) args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -89,18 +89,18 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def normalize_templates(templates: list[str]) -> list[str]: - normalized_templates = list(templates) + templates = list(templates) - if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: - django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) - normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + if TemplateType.DJANGO_FASTAPI in templates and TemplateType.WEBAPP not in templates: + django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) + templates.insert(django_app_index, TemplateType.WEBAPP) - has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) - if TemplateType.DJANGO_APP in normalized_templates and not has_database_template: - django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) - normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + has_database_template = any(template in TemplateType.database_templates() for template in templates) + if TemplateType.DJANGO_FASTAPI in templates and not has_database_template: + django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) + templates.insert(django_app_index, TemplateType.DB_POSTGRES) - return normalized_templates + return templates def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index c51edf25..00b80019 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -164,7 +164,7 @@ def generate_servers( if not should_generate(f'server stubs for {openapi_file}'): continue - if TemplateType.DJANGO_APP in manifest.templates: + if TemplateType.DJANGO_FASTAPI in manifest.templates: generate_fastapi_server(app_path) if TemplateType.FLASK_SERVER in manifest.templates: @@ -279,12 +279,14 @@ def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: try: manifest_data = load_yaml(manifest_file) + manifest_data, update_manifest = CloudHarnessManifest.migrate(manifest_data) + if update_manifest: + save_yaml(manifest_file, manifest_data) manifest = CloudHarnessManifest.from_dict(manifest_data) except (FileNotFoundError, YAMLError): logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') manifest = CloudHarnessManifest( app_name=app_path.name, - version='1', inferred=True, templates=infer_templates(app_path), ) @@ -313,7 +315,7 @@ def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: genapi_path = app_path / 'api' / 'genapi.sh' if genapi_path.exists(): - templates.append(TemplateType.DJANGO_APP) + templates.append(TemplateType.DJANGO_FASTAPI) return server_path = app_path / 'server' From 85b863a31b2abeb6ab342e7a4e6ebc68b697160e Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 12:55:45 +0000 Subject: [PATCH 03/15] CH-157 Add app builder for django-ninja applications using a shared base with django-fastapi --- .../{django-app => django-base}/.dockerignore | 0 .../api/test_st.py | 0 .../backend/__APP_NAME__/__init__.py | 0 .../backend/__APP_NAME__/admin.py | 0 .../backend/__APP_NAME__/apps.py | 0 .../__APP_NAME__/migrations/0001_initial.py | 0 .../__APP_NAME__/migrations/__init__.py | 0 .../backend/__APP_NAME__/models.py | 0 .../backend/__APP_NAME__/tests.py | 0 .../backend/__APP_NAME__/views.py | 0 .../backend/django_baseapp/__init__.py | 0 .../backend/django_baseapp/asgi.py | 0 .../django_baseapp/migrations/0001_initial.py | 0 .../django_baseapp/migrations/__init__.py | 0 .../backend/django_baseapp/models.py | 0 .../django_baseapp/static/www/index.html | 0 .../templates/__APP_NAME__/index.html | 0 .../templates/__APP_NAME__/swagger-ui.html | 0 .../backend/django_baseapp/views.py | 0 .../backend/django_baseapp/wsgi.py | 0 .../backend/manage.py | 0 .../backend/openapi/.gitkeep | 0 .../backend/persistent/.gitkeep | 0 .../backend/setup.py | 0 .../backend/static/www/.gitkeep | 0 .../deploy/values.yaml | 0 .../{django-app => django-fastapi}/Dockerfile | 0 .../{django-app => django-fastapi}/README.md | 0 .../api/genapi.sh | 0 .../api/openapi.yaml | 0 .../api/templates/main.jinja2 | 0 .../backend/README.md | 0 .../__APP_NAME__/controllers/__init__.py | 0 .../backend/__APP_NAME__/controllers/test.py | 0 .../backend/django_baseapp/settings.py | 0 .../backend/django_baseapp/urls.py | 0 .../backend/requirements.txt | 0 .../dev-setup.sh | 0 application-templates/django-ninja/Dockerfile | 35 ++++ application-templates/django-ninja/README.md | 86 +++++++++ .../backend/__APP_NAME__/api/__init__.py | 40 +++++ .../backend/__APP_NAME__/exceptions.py | 6 + .../backend/__APP_NAME__/schema.py | 3 + .../backend/django_baseapp/settings.py | 167 ++++++++++++++++++ .../backend/django_baseapp/urls.py | 31 ++++ .../django-ninja/backend/requirements.txt | 2 + .../django-ninja/dev-setup.sh | 50 ++++++ .../ch_cli_tools/application_builders.py | 75 +++++--- .../ch_cli_tools/common_types.py | 5 + .../deployment-cli-tools/harness-application | 15 +- tools/deployment-cli-tools/harness-generate | 15 +- 51 files changed, 496 insertions(+), 34 deletions(-) rename application-templates/{django-app => django-base}/.dockerignore (100%) rename application-templates/{django-app => django-base}/api/test_st.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/admin.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/apps.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/migrations/0001_initial.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/migrations/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/models.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/tests.py (100%) rename application-templates/{django-app => django-base}/backend/__APP_NAME__/views.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/asgi.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/migrations/0001_initial.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/migrations/__init__.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/models.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/static/www/index.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/templates/__APP_NAME__/index.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/views.py (100%) rename application-templates/{django-app => django-base}/backend/django_baseapp/wsgi.py (100%) rename application-templates/{django-app => django-base}/backend/manage.py (100%) rename application-templates/{django-app => django-base}/backend/openapi/.gitkeep (100%) rename application-templates/{django-app => django-base}/backend/persistent/.gitkeep (100%) rename application-templates/{django-app => django-base}/backend/setup.py (100%) rename application-templates/{django-app => django-base}/backend/static/www/.gitkeep (100%) rename application-templates/{django-app => django-base}/deploy/values.yaml (100%) rename application-templates/{django-app => django-fastapi}/Dockerfile (100%) rename application-templates/{django-app => django-fastapi}/README.md (100%) rename application-templates/{django-app => django-fastapi}/api/genapi.sh (100%) rename application-templates/{django-app => django-fastapi}/api/openapi.yaml (100%) rename application-templates/{django-app => django-fastapi}/api/templates/main.jinja2 (100%) rename application-templates/{django-app => django-fastapi}/backend/README.md (100%) rename application-templates/{django-app => django-fastapi}/backend/__APP_NAME__/controllers/__init__.py (100%) rename application-templates/{django-app => django-fastapi}/backend/__APP_NAME__/controllers/test.py (100%) rename application-templates/{django-app => django-fastapi}/backend/django_baseapp/settings.py (100%) rename application-templates/{django-app => django-fastapi}/backend/django_baseapp/urls.py (100%) rename application-templates/{django-app => django-fastapi}/backend/requirements.txt (100%) rename application-templates/{django-app => django-fastapi}/dev-setup.sh (100%) create mode 100644 application-templates/django-ninja/Dockerfile create mode 100644 application-templates/django-ninja/README.md create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/exceptions.py create mode 100644 application-templates/django-ninja/backend/__APP_NAME__/schema.py create mode 100644 application-templates/django-ninja/backend/django_baseapp/settings.py create mode 100644 application-templates/django-ninja/backend/django_baseapp/urls.py create mode 100644 application-templates/django-ninja/backend/requirements.txt create mode 100644 application-templates/django-ninja/dev-setup.sh diff --git a/application-templates/django-app/.dockerignore b/application-templates/django-base/.dockerignore similarity index 100% rename from application-templates/django-app/.dockerignore rename to application-templates/django-base/.dockerignore diff --git a/application-templates/django-app/api/test_st.py b/application-templates/django-base/api/test_st.py similarity index 100% rename from application-templates/django-app/api/test_st.py rename to application-templates/django-base/api/test_st.py diff --git a/application-templates/django-app/backend/__APP_NAME__/__init__.py b/application-templates/django-base/backend/__APP_NAME__/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/admin.py b/application-templates/django-base/backend/__APP_NAME__/admin.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/admin.py rename to application-templates/django-base/backend/__APP_NAME__/admin.py diff --git a/application-templates/django-app/backend/__APP_NAME__/apps.py b/application-templates/django-base/backend/__APP_NAME__/apps.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/apps.py rename to application-templates/django-base/backend/__APP_NAME__/apps.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py b/application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/0001_initial.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py b/application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/migrations/__init__.py rename to application-templates/django-base/backend/__APP_NAME__/migrations/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/models.py b/application-templates/django-base/backend/__APP_NAME__/models.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/models.py rename to application-templates/django-base/backend/__APP_NAME__/models.py diff --git a/application-templates/django-app/backend/__APP_NAME__/tests.py b/application-templates/django-base/backend/__APP_NAME__/tests.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/tests.py rename to application-templates/django-base/backend/__APP_NAME__/tests.py diff --git a/application-templates/django-app/backend/__APP_NAME__/views.py b/application-templates/django-base/backend/__APP_NAME__/views.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/views.py rename to application-templates/django-base/backend/__APP_NAME__/views.py diff --git a/application-templates/django-app/backend/django_baseapp/__init__.py b/application-templates/django-base/backend/django_baseapp/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/__init__.py rename to application-templates/django-base/backend/django_baseapp/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/asgi.py rename to application-templates/django-base/backend/django_baseapp/asgi.py diff --git a/application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py b/application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/0001_initial.py rename to application-templates/django-base/backend/django_baseapp/migrations/0001_initial.py diff --git a/application-templates/django-app/backend/django_baseapp/migrations/__init__.py b/application-templates/django-base/backend/django_baseapp/migrations/__init__.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/migrations/__init__.py rename to application-templates/django-base/backend/django_baseapp/migrations/__init__.py diff --git a/application-templates/django-app/backend/django_baseapp/models.py b/application-templates/django-base/backend/django_baseapp/models.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/models.py rename to application-templates/django-base/backend/django_baseapp/models.py diff --git a/application-templates/django-app/backend/django_baseapp/static/www/index.html b/application-templates/django-base/backend/django_baseapp/static/www/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/static/www/index.html rename to application-templates/django-base/backend/django_baseapp/static/www/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/index.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/index.html diff --git a/application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html b/application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html similarity index 100% rename from application-templates/django-app/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html rename to application-templates/django-base/backend/django_baseapp/templates/__APP_NAME__/swagger-ui.html diff --git a/application-templates/django-app/backend/django_baseapp/views.py b/application-templates/django-base/backend/django_baseapp/views.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/views.py rename to application-templates/django-base/backend/django_baseapp/views.py diff --git a/application-templates/django-app/backend/django_baseapp/wsgi.py b/application-templates/django-base/backend/django_baseapp/wsgi.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/wsgi.py rename to application-templates/django-base/backend/django_baseapp/wsgi.py diff --git a/application-templates/django-app/backend/manage.py b/application-templates/django-base/backend/manage.py similarity index 100% rename from application-templates/django-app/backend/manage.py rename to application-templates/django-base/backend/manage.py diff --git a/application-templates/django-app/backend/openapi/.gitkeep b/application-templates/django-base/backend/openapi/.gitkeep similarity index 100% rename from application-templates/django-app/backend/openapi/.gitkeep rename to application-templates/django-base/backend/openapi/.gitkeep diff --git a/application-templates/django-app/backend/persistent/.gitkeep b/application-templates/django-base/backend/persistent/.gitkeep similarity index 100% rename from application-templates/django-app/backend/persistent/.gitkeep rename to application-templates/django-base/backend/persistent/.gitkeep diff --git a/application-templates/django-app/backend/setup.py b/application-templates/django-base/backend/setup.py similarity index 100% rename from application-templates/django-app/backend/setup.py rename to application-templates/django-base/backend/setup.py diff --git a/application-templates/django-app/backend/static/www/.gitkeep b/application-templates/django-base/backend/static/www/.gitkeep similarity index 100% rename from application-templates/django-app/backend/static/www/.gitkeep rename to application-templates/django-base/backend/static/www/.gitkeep diff --git a/application-templates/django-app/deploy/values.yaml b/application-templates/django-base/deploy/values.yaml similarity index 100% rename from application-templates/django-app/deploy/values.yaml rename to application-templates/django-base/deploy/values.yaml diff --git a/application-templates/django-app/Dockerfile b/application-templates/django-fastapi/Dockerfile similarity index 100% rename from application-templates/django-app/Dockerfile rename to application-templates/django-fastapi/Dockerfile diff --git a/application-templates/django-app/README.md b/application-templates/django-fastapi/README.md similarity index 100% rename from application-templates/django-app/README.md rename to application-templates/django-fastapi/README.md diff --git a/application-templates/django-app/api/genapi.sh b/application-templates/django-fastapi/api/genapi.sh similarity index 100% rename from application-templates/django-app/api/genapi.sh rename to application-templates/django-fastapi/api/genapi.sh diff --git a/application-templates/django-app/api/openapi.yaml b/application-templates/django-fastapi/api/openapi.yaml similarity index 100% rename from application-templates/django-app/api/openapi.yaml rename to application-templates/django-fastapi/api/openapi.yaml diff --git a/application-templates/django-app/api/templates/main.jinja2 b/application-templates/django-fastapi/api/templates/main.jinja2 similarity index 100% rename from application-templates/django-app/api/templates/main.jinja2 rename to application-templates/django-fastapi/api/templates/main.jinja2 diff --git a/application-templates/django-app/backend/README.md b/application-templates/django-fastapi/backend/README.md similarity index 100% rename from application-templates/django-app/backend/README.md rename to application-templates/django-fastapi/backend/README.md diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/__init__.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/__init__.py diff --git a/application-templates/django-app/backend/__APP_NAME__/controllers/test.py b/application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py similarity index 100% rename from application-templates/django-app/backend/__APP_NAME__/controllers/test.py rename to application-templates/django-fastapi/backend/__APP_NAME__/controllers/test.py diff --git a/application-templates/django-app/backend/django_baseapp/settings.py b/application-templates/django-fastapi/backend/django_baseapp/settings.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/settings.py rename to application-templates/django-fastapi/backend/django_baseapp/settings.py diff --git a/application-templates/django-app/backend/django_baseapp/urls.py b/application-templates/django-fastapi/backend/django_baseapp/urls.py similarity index 100% rename from application-templates/django-app/backend/django_baseapp/urls.py rename to application-templates/django-fastapi/backend/django_baseapp/urls.py diff --git a/application-templates/django-app/backend/requirements.txt b/application-templates/django-fastapi/backend/requirements.txt similarity index 100% rename from application-templates/django-app/backend/requirements.txt rename to application-templates/django-fastapi/backend/requirements.txt diff --git a/application-templates/django-app/dev-setup.sh b/application-templates/django-fastapi/dev-setup.sh similarity index 100% rename from application-templates/django-app/dev-setup.sh rename to application-templates/django-fastapi/dev-setup.sh diff --git a/application-templates/django-ninja/Dockerfile b/application-templates/django-ninja/Dockerfile new file mode 100644 index 00000000..f13bfaac --- /dev/null +++ b/application-templates/django-ninja/Dockerfile @@ -0,0 +1,35 @@ +ARG MNP_UI +ARG CLOUDHARNESS_DJANGO + +FROM $MNP_UI AS frontend + +ARG APP_DIR=/app + +WORKDIR ${APP_DIR} +COPY frontend/package.json . +COPY frontend/yarn.lock . +RUN yarn install --timeout 60000 + +COPY frontend . +RUN yarn build + +##### + +FROM $CLOUDHARNESS_DJANGO + +WORKDIR ${APP_DIR} +RUN mkdir -p ${APP_DIR}/static/www + +COPY backend/requirements.txt ${APP_DIR} +RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ + pip3 install --no-cache-dir -r requirements.txt --prefer-binary + +COPY backend/requirements.txt backend/setup.py ${APP_DIR} +RUN python3 -m pip install -e . + +COPY backend ${APP_DIR} +RUN python3 manage.py collectstatic --noinput + +COPY --from=frontend /app/dist ${APP_DIR}/static/www + +ENTRYPOINT uvicorn --workers ${WORKERS} --host 0.0.0.0 --port ${PORT} django_baseapp.asgi:application diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md new file mode 100644 index 00000000..cc4b4c09 --- /dev/null +++ b/application-templates/django-ninja/README.md @@ -0,0 +1,86 @@ +# __APP_NAME__ + +Django-Ninja/React-based web application. +This application is constructed to be deployed inside a cloud-harness Kubernetes. +It can be also run locally for development and test purpose. + +The code is generated with the script `harness-application`. + +## Configuration + +### Accounts + +The CloudHarness Django application template comes with a configuration that can retrieve user account updates from Keycloak (accounts) +To enable this feature: +* log in into the accounts admin interface +* select in the left sidebar Events +* select the `Config` tab +* enable "metacell-admin-event-listener" under the `Events Config` - `Event Listeners` + +An other option is to enable the "metacell-admin-event-listener" through customizing the Keycloak realm.json from the CloudHarness repository. + +## Develop + +This application is composed of a Django-Ninja backend and a React frontend. + +### Backend + +Backend code is inside the *backend* directory. +See [backend/README.md#Develop] + +### Frontend + +Backend code is inside the *frontend* directory. + +Frontend is by default generated as a React web application, but no constraint about this specific technology. + +#### Call the backend apis +All the api stubs are automatically generated in the [frontend/rest](frontend/rest) directory by `harness-application` +and `harness-generate`. + +## Local build & run + +### Install dependencies +1 - Clone cloud-harness into your project root folder + +2 - Run the dev setup script +``` +cd applications/__APP_NAME__ +bash dev-setup.sh +``` + +### Prepare backend + +Create a Django local superuser account, this you only need to do on initial setup +```bash +cd backend +python3 manage.py migrate # to sync the database with the Django models +python3 manage.py collectstatic --noinput # to copy all assets to the static folder +python3 manage.py createsuperuser +# link the frontend dist to the django static folder, this is only needed once, frontend updates will automatically be applied +cd static/www +ln -s ../../../frontend/dist dist +``` + +### Build frontend + +Compile the frontend +```bash +cd frontend +npm install +npm run build +``` + +### Run backend application + +start the Django server +```bash +uvicorn --workers 2 --host 0.0.0.0 --port 8000 django_baseapp.asgi:application +``` + + +### Running local with port forwardings to a kubernetes cluster +When you create port forwards to microservices in your k8s cluster you want to forced your local backend server to initialize +the AuthService and EventService services. +This can be done by setting the `KUBERNETES_SERVICE_HOST` environment variable to a dummy or correct k8s service host. +The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak client and client roles of this microservice. diff --git a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py new file mode 100644 index 00000000..647697a0 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py @@ -0,0 +1,40 @@ +import time +from django.http import HttpRequest +from ninja import NinjaAPI +from ..exceptions import Http401, Http403 + + +api = NinjaAPI(title='__APP_NAME__ API', version='0.1.0') + + +@api.exception_handler(Http401) +def unauthorized(request, exc): + return api.create_response( + request, + {'message': 'Unauthorized'}, + status=401, + ) + + +@api.exception_handler(Http403) +def forbidden(request, exc): + return api.create_response( + request, + {'message': 'Forbidden'}, + status=403, + ) + + +@api.get('/ping', response={200: float}) +def ping(request: HttpRequest): + return time.time() + + +@api.get('/live', response={200: str}) +def live(request: HttpRequest): + return 'OK' + + +@api.get('/ready', response={200: str}) +def ready(request: HttpRequest): + return 'OK' diff --git a/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py new file mode 100644 index 00000000..b5b053c1 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/exceptions.py @@ -0,0 +1,6 @@ +class Http401(Exception): + pass + + +class Http403(Exception): + pass diff --git a/application-templates/django-ninja/backend/__APP_NAME__/schema.py b/application-templates/django-ninja/backend/__APP_NAME__/schema.py new file mode 100644 index 00000000..b3d9c8b3 --- /dev/null +++ b/application-templates/django-ninja/backend/__APP_NAME__/schema.py @@ -0,0 +1,3 @@ +from ninja import Schema + +# Create your schema here diff --git a/application-templates/django-ninja/backend/django_baseapp/settings.py b/application-templates/django-ninja/backend/django_baseapp/settings.py new file mode 100644 index 00000000..e54f20f0 --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/settings.py @@ -0,0 +1,167 @@ +""" +Django settings for the MNP Checkout project. + +Generated by 'django-admin startproject' using Django 3.2.12. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-81kv$0=07xac7r(pgz6ndb5t0at4-z@ae6&f@u6_3jo&9d#4kl" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False if os.environ.get("PRODUCTION", None) else True + +ALLOWED_HOSTS = [ + "*", +] + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + 'django.middleware.csrf.CsrfViewMiddleware', + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "cloudharness.middleware.django.CloudharnessMiddleware", +] + + +ROOT_URLCONF = "django_baseapp.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [BASE_DIR / "templates"], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "django_baseapp.wsgi.application" + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + + +PROJECT_NAME = "__APP_NAME__".upper() + +# Persistent storage +PERSISTENT_ROOT = os.path.join(BASE_DIR, "persistent") + +# *********************************************************************** +# * __APP_NAME__ settings +# *********************************************************************** +from cloudharness.applications import get_configuration # noqa E402 +from cloudharness.utils.config import ALLVALUES_PATH, CloudharnessConfig # noqa E402 + +# *********************************************************************** +# * import base CloudHarness Django settings +# *********************************************************************** +from cloudharness_django.settings import * # noqa E402 + +# add the local apps +INSTALLED_APPS += [ + "__APP_NAME__", + "django_baseapp", + "ninja", +] + +# override django admin base template with a local template +# to add some custom styling +TEMPLATES[0]["DIRS"] = [BASE_DIR / "templates"] + +# Static files (CSS, JavaScript, Images) +MEDIA_ROOT = PERSISTENT_ROOT +STATIC_ROOT = os.path.join(BASE_DIR, "static") +MEDIA_URL = "/media/" +STATIC_URL = "/static/" + +# KC Client & roles +KC_CLIENT_NAME = PROJECT_NAME.lower() + +# __APP_NAME__ specific roles + +# Default KC roles +KC_ADMIN_ROLE = f"{KC_CLIENT_NAME}-administrator" # admin user +KC_MANAGER_ROLE = f"{KC_CLIENT_NAME}-manager" # manager user +KC_USER_ROLE = f"{KC_CLIENT_NAME}-user" # customer user +KC_ALL_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, + KC_USER_ROLE, +] +KC_PRIVILEGED_ROLES = [ + KC_ADMIN_ROLE, + KC_MANAGER_ROLE, +] + +KC_DEFAULT_USER_ROLE = None # don't add the user role to the realm default role diff --git a/application-templates/django-ninja/backend/django_baseapp/urls.py b/application-templates/django-ninja/backend/django_baseapp/urls.py new file mode 100644 index 00000000..aae3d635 --- /dev/null +++ b/application-templates/django-ninja/backend/django_baseapp/urls.py @@ -0,0 +1,31 @@ +"""MNP Checkout URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +import re +from django.conf import settings +from django.views.static import serve +from django.contrib import admin +from django.urls import path, re_path +from __APP_NAME__.api import api +from django_baseapp.views import index + + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", api.urls), + re_path(r"^%s(?P.*)$" % re.escape(settings.MEDIA_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.MEDIA_ROOT)), + re_path(r"^%s(?P.*)$" % re.escape(settings.STATIC_URL.lstrip("/")), serve, kwargs=dict(document_root=settings.STATIC_ROOT)), + re_path(r"^(?P.*)$", index, name="index"), +] diff --git a/application-templates/django-ninja/backend/requirements.txt b/application-templates/django-ninja/backend/requirements.txt new file mode 100644 index 00000000..97c58008 --- /dev/null +++ b/application-templates/django-ninja/backend/requirements.txt @@ -0,0 +1,2 @@ +pydantic==2.9.2 +ninja \ No newline at end of file diff --git a/application-templates/django-ninja/dev-setup.sh b/application-templates/django-ninja/dev-setup.sh new file mode 100644 index 00000000..24970bf3 --- /dev/null +++ b/application-templates/django-ninja/dev-setup.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$(pwd) +CH_DIRECTORY="../../cloud-harness" +INSTALL_PYTEST=false +CURRENT_DIRECTORY="$(pwd)" +APP_NAME="__APP_NAME__" + +pip_upgrade_error() { + echo "Unable to upgrade pip" + exit 1 +} + +install_error () { + echo "Unable to install $1" 1>&2 + exit 1 +} + +while getopts ch_directory:pytest arg; +do + case "$arg" in + ch_directory) CH_DIRECTORY=${OPTARG};; + pytest) INSTALL_PYTEST=true;; + esac +done + +pip install --upgrade pip || pip_upgrade_error + +# Install pip dependencies from cloudharness-base-debian image + +if $INSTALL_PYTEST; then + pip install pytest || install_error pytest +fi + +pip install -r "$CH_DIRECTORY/libraries/models/requirements.txt" || install_error "models requirements" +pip install -r "$CH_DIRECTORY/libraries/cloudharness-common/requirements.txt" || install_error "cloudharness-common requirements" +pip install -r "$CH_DIRECTORY/libraries/client/cloudharness_cli/requirements.txt" || install_error "cloudharness_cli requirements" + +pip install -e "$CH_DIRECTORY/libraries/models" || install_error models +pip install -e "$CH_DIRECTORY/libraries/cloudharness-common" || install_error cloudharness-common +pip install -e "$CH_DIRECTORY/libraries/client/cloudharness_cli" || install_error cloudharness_cli + +# Install pip dependencies from cloudharness-django image + +pip install -e "$CH_DIRECTORY/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django" || install_error cloudharness-django + +# Install application + +pip install -r "$CURRENT_DIRECTORY/backend/requirements.txt" || install_error "$APP_NAME dependencies" +pip install -e "$CURRENT_DIRECTORY/backend" || install_error "$APP_NAME" \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 0f5bca0e..611955b5 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -7,7 +7,7 @@ import tempfile from .common_types import TemplateType from .openapi import generate_fastapi_server, generate_server, generate_ts_client -from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, to_python_module +from .utils import copymergedir, get_json_template, merge_configuration_directories, replace_in_dict, replace_in_file, replaceindir, to_python_module from . import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH @@ -134,25 +134,13 @@ def handle_post_merge(self): generate_server(self.app_path) -class DjangoFastApiBuilder(ApplicationBuilder): - def handles(self, templates): - return TemplateType.DJANGO_FASTAPI in templates - - def handle_pre_merge(self): - pass - +class BaseDjangoAppBuilder(ApplicationBuilder): + @abc.abstractmethod def handle_merge(self): - self.merge_template_directories(TemplateType.DJANGO_FASTAPI) + self.merge_template_directories('django-base') + @abc.abstractmethod def handle_post_merge(self): - replace_in_file( - self.api_path / 'templates' / 'main.jinja2', - self.APP_NAME_PLACEHOLDER, - self.python_app_name, - ) - replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) - generate_fastapi_server(self.app_path) - replace_in_file( self.app_path / 'deploy' / 'values.yaml', f'{self.APP_NAME_PLACEHOLDER}:{self.APP_NAME_PLACEHOLDER}', @@ -161,12 +149,6 @@ def handle_post_merge(self): replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) self.create_django_app_vscode_debug_configuration() - - (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) - - @property - def python_app_name(self): - return to_python_module(self.app_name) def create_django_app_vscode_debug_configuration(self): vscode_launch_path = pathlib.Path('.vscode/launch.json') @@ -188,6 +170,50 @@ def create_django_app_vscode_debug_configuration(self): with vscode_launch_path.open('w') as f: json.dump(launch_config, f, indent=2, sort_keys=True) + @property + def python_app_name(self): + return to_python_module(self.app_name) + + +class DjangoFastApiBuilder(BaseDjangoAppBuilder): + def handles(self, templates): + return TemplateType.DJANGO_FASTAPI in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_FASTAPI) + + def handle_post_merge(self): + super().handle_post_merge() + + replace_in_file( + self.api_path / 'templates' / 'main.jinja2', + self.APP_NAME_PLACEHOLDER, + self.python_app_name, + ) + replace_in_file(self.api_path / 'genapi.sh', self.APP_NAME_PLACEHOLDER, self.app_name) + generate_fastapi_server(self.app_path) + + (self.backend_path / self.APP_NAME_PLACEHOLDER / '__main__.py').unlink(missing_ok=True) + + +class DjangoNinjaBuilder(BaseDjangoAppBuilder): + def handles(self, templates): + return TemplateType.DJANGO_NINJA in templates + + def handle_pre_merge(self): + pass + + def handle_merge(self): + super().handle_merge() + self.merge_template_directories(TemplateType.DJANGO_NINJA) + + def handle_post_merge(self): + super().handle_post_merge() + class AppBuilderPipeline(ApplicationBuilder): def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): @@ -198,6 +224,7 @@ def __init__(self, app_name: str, app_path: pathlib.Path, templates: list[str]): TemplateType.SERVER: ServerAppBuilder(app_name, app_path), TemplateType.FLASK_SERVER: FlaskServerAppBuilder(app_name, app_path), TemplateType.DJANGO_FASTAPI: DjangoFastApiBuilder(app_name, app_path), + TemplateType.DJANGO_NINJA: DjangoNinjaBuilder(app_name, app_path), } def handles(self, templates): @@ -207,6 +234,7 @@ def handle_pre_merge(self): pre_merge_template_order = [ TemplateType.FLASK_SERVER, TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, TemplateType.WEBAPP, TemplateType.SERVER, ] @@ -232,6 +260,7 @@ def handle_post_merge(self): post_merge_template_order = [ TemplateType.FLASK_SERVER, TemplateType.DJANGO_FASTAPI, + TemplateType.DJANGO_NINJA, TemplateType.WEBAPP, TemplateType.SERVER, ] diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 99b9d41c..b88a19c9 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -17,12 +17,17 @@ class TemplateType(StrEnum): DB_NEO4J = 'db-neo4j' DB_MONGO = 'db-mongo' DJANGO_FASTAPI = 'django-fastapi' + DJANGO_NINJA = 'django-ninja' SERVER = 'server' @classmethod def database_templates(cls): return [cls.DB_POSTGRES, cls.DB_NEO4J, cls.DB_MONGO] + @classmethod + def django_templates(cls) -> list[str]: + return [cls.DJANGO_FASTAPI, cls.DJANGO_FASTAPI] + @dataclass class CloudHarnessManifest(): diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index 2a65fe37..7484f754 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -66,6 +66,7 @@ def get_command_line_arguments() -> tuple[str, list[str]]: - db-neo4j - db-mongo - django-fastapi (fastapi django backend based on openapi) + - django-ninja (django ninja backend) """) args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -91,14 +92,16 @@ def get_command_line_arguments() -> tuple[str, list[str]]: def normalize_templates(templates: list[str]) -> list[str]: templates = list(templates) - if TemplateType.DJANGO_FASTAPI in templates and TemplateType.WEBAPP not in templates: - django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) - templates.insert(django_app_index, TemplateType.WEBAPP) + def django_template_index(): + return next(index for index, template in enumerate(templates) if template in TemplateType.django_templates()) + + has_django_template = any(template in TemplateType.django_templates() for template in templates) + if has_django_template and TemplateType.WEBAPP not in templates: + templates.insert(django_template_index(), TemplateType.WEBAPP) has_database_template = any(template in TemplateType.database_templates() for template in templates) - if TemplateType.DJANGO_FASTAPI in templates and not has_database_template: - django_app_index = templates.index(TemplateType.DJANGO_FASTAPI) - templates.insert(django_app_index, TemplateType.DB_POSTGRES) + if has_django_template and not has_database_template: + templates.insert(django_template_index(), TemplateType.DB_POSTGRES) return templates diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 00b80019..8cf6e466 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -312,14 +312,19 @@ def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: - genapi_path = app_path / 'api' / 'genapi.sh' - - if genapi_path.exists(): - templates.append(TemplateType.DJANGO_FASTAPI) + backend_path = app_path / 'backend' + manage_path = backend_path / 'manage.py' + + if manage_path.exists(): + requirements_path = backend_path / 'requirements.txt' + requirements = requirements_path.read_text() + if 'ninja' in requirements: + templates.append(TemplateType.DJANGO_NINJA) + else: + templates.append(TemplateType.DJANGO_FASTAPI) return server_path = app_path / 'server' - backend_path = app_path / 'backend' if server_path.exists() or backend_path.exists(): templates.append(TemplateType.FLASK_SERVER) From 5b790444ebc4b84bd8b6699186e80d467ee405e4 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 16:18:15 +0000 Subject: [PATCH 04/15] CH-157 Add specific debug templates for the two different djano templates --- ...vscode-django-fastapi-debug-template.json} | 0 .../vscode-django-ninja-debug-template.json | 24 +++++++++++++++++++ .../ch_cli_tools/application_builders.py | 11 ++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) rename deployment-configuration/{vscode-django-app-debug-template.json => vscode-django-fastapi-debug-template.json} (100%) create mode 100644 deployment-configuration/vscode-django-ninja-debug-template.json diff --git a/deployment-configuration/vscode-django-app-debug-template.json b/deployment-configuration/vscode-django-fastapi-debug-template.json similarity index 100% rename from deployment-configuration/vscode-django-app-debug-template.json rename to deployment-configuration/vscode-django-fastapi-debug-template.json diff --git a/deployment-configuration/vscode-django-ninja-debug-template.json b/deployment-configuration/vscode-django-ninja-debug-template.json new file mode 100644 index 00000000..198d465d --- /dev/null +++ b/deployment-configuration/vscode-django-ninja-debug-template.json @@ -0,0 +1,24 @@ +{ + "args": [ + "--host", + "0.0.0.0", + "--port", + "8000", + "django_baseapp.asgi:application" + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/applications/__APP_NAME__/backend", + "env": { + "ACCOUNTS_ADMIN_PASSWORD": "metacell", + "ACCOUNTS_ADMIN_USERNAME": "admin", + "CH_CURRENT_APP_NAME": "__APP_NAME__", + "CH_VALUES_PATH": "${workspaceFolder}/deployment/helm/values.yaml", + "DJANGO_SETTINGS_MODULE": "django_baseapp.settings", + "KUBERNETES_SERVICE_HOST": "ssdds" + }, + "justMyCode": false, + "module": "uvicorn", + "name": "__APP_NAME__ backend", + "request": "launch", + "type": "debugpy" + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index 611955b5..e58ff681 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -161,7 +161,7 @@ def create_django_app_vscode_debug_configuration(self): if configuration['name'] != configuration_name ] - debug_config = get_json_template('vscode-django-app-debug-template.json', True) + debug_config = get_json_template(self.debug_template_file, True) debug_config = replace_in_dict(debug_config, self.APP_NAME_PLACEHOLDER, self.app_name) launch_config['configurations'].append(debug_config) @@ -173,9 +173,16 @@ def create_django_app_vscode_debug_configuration(self): @property def python_app_name(self): return to_python_module(self.app_name) + + @property + @abc.abstractmethod + def debug_template_file(self) -> str: + raise NotImplementedError() class DjangoFastApiBuilder(BaseDjangoAppBuilder): + debug_template_name = 'vscode-django-fastapi-debug-template.json' + def handles(self, templates): return TemplateType.DJANGO_FASTAPI in templates @@ -201,6 +208,8 @@ def handle_post_merge(self): class DjangoNinjaBuilder(BaseDjangoAppBuilder): + debug_template_name = 'vscode-django-ninja-debug-template.json' + def handles(self, templates): return TemplateType.DJANGO_NINJA in templates From 025d5c80169628cdb5c091be9fa8a3e8ea882da6 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:56:40 +0000 Subject: [PATCH 05/15] CH-157 Fix errors in django-ninja template --- .../django-ninja/backend/__APP_NAME__/api/__init__.py | 6 +++--- application-templates/django-ninja/backend/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py index 647697a0..ab1fdac8 100644 --- a/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py +++ b/application-templates/django-ninja/backend/__APP_NAME__/api/__init__.py @@ -25,16 +25,16 @@ def forbidden(request, exc): ) -@api.get('/ping', response={200: float}) +@api.get('/ping', response={200: float}, tags=['test']) def ping(request: HttpRequest): return time.time() -@api.get('/live', response={200: str}) +@api.get('/live', response={200: str}, tags=['test']) def live(request: HttpRequest): return 'OK' -@api.get('/ready', response={200: str}) +@api.get('/ready', response={200: str}, tags=['test']) def ready(request: HttpRequest): return 'OK' diff --git a/application-templates/django-ninja/backend/requirements.txt b/application-templates/django-ninja/backend/requirements.txt index 97c58008..b3575416 100644 --- a/application-templates/django-ninja/backend/requirements.txt +++ b/application-templates/django-ninja/backend/requirements.txt @@ -1,2 +1,2 @@ pydantic==2.9.2 -ninja \ No newline at end of file +django-ninja \ No newline at end of file From 46488238f381ce6265d67a6f0f7016808055ade7 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:57:20 +0000 Subject: [PATCH 06/15] CH-157 Add utility method to run djano-ninja openapi export --- tools/deployment-cli-tools/ch_cli_tools/openapi.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 7de31d8c..fcc91974 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -124,6 +124,20 @@ def generate_ts_client(openapi_file): replaceindir(out_dir, "http://localhost", '') +def generate_openapi_from_ninja_schema(app_name: str, app_path: pathlib.Path) -> None: + out_path = app_path / 'api' / 'openapi.yaml' + manage_path = app_path / 'backend' / 'manage.py' + command = [ + 'python', manage_path, 'export_openapi_schema', + '--settings', 'django_baseapp.settings', + '--api', f'{to_python_module(app_name)}.api.api', + '--output', out_path, + '--indent', '2', + ] + + subprocess.run(command) + + def get_dependencies(): """ Checks if java is installed From 9febcd62a970206f1d10b0cf03d44c3298c6cb0c Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:57:58 +0000 Subject: [PATCH 07/15] CH-157 Fix bug with django ninja not considered a django template --- tools/deployment-cli-tools/ch_cli_tools/common_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index b88a19c9..6c8e2c4c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -26,7 +26,7 @@ def database_templates(cls): @classmethod def django_templates(cls) -> list[str]: - return [cls.DJANGO_FASTAPI, cls.DJANGO_FASTAPI] + return [cls.DJANGO_FASTAPI, cls.DJANGO_NINJA] @dataclass From 33d9803df9c45db9525ccf50bbe266df1bbdff34 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:58:42 +0000 Subject: [PATCH 08/15] CH-157 Fix bug with incorrectly named field in derived app builders --- .../deployment-cli-tools/ch_cli_tools/application_builders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index e58ff681..ef020633 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -181,7 +181,7 @@ def debug_template_file(self) -> str: class DjangoFastApiBuilder(BaseDjangoAppBuilder): - debug_template_name = 'vscode-django-fastapi-debug-template.json' + debug_template_file = 'vscode-django-fastapi-debug-template.json' def handles(self, templates): return TemplateType.DJANGO_FASTAPI in templates @@ -208,7 +208,7 @@ def handle_post_merge(self): class DjangoNinjaBuilder(BaseDjangoAppBuilder): - debug_template_name = 'vscode-django-ninja-debug-template.json' + debug_template_file = 'vscode-django-ninja-debug-template.json' def handles(self, templates): return TemplateType.DJANGO_NINJA in templates From dca0eae2c16a52f0990b3eb2f0ad66e9acae6d2d Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 17:59:42 +0000 Subject: [PATCH 09/15] CH-157 Update harness-generate to leverage django-ninja schema generation --- tools/deployment-cli-tools/harness-generate | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 8cf6e466..545e6612 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -11,7 +11,7 @@ import logging from typing import Callable, Optional from ruamel.yaml.error import YAMLError -from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ +from ch_cli_tools.openapi import LIB_NAME, generate_openapi_from_ninja_schema, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType @@ -184,21 +184,25 @@ def generate_clients( if not should_generate('client libraries'): return - openapi_files = get_openapi_file_paths(root_path) client_src_path = root_path / 'libraries' / 'client' / client_lib_name + apps_path = root_path / 'applications' + apps = (app for app in apps_path.iterdir() if app.is_dir()) - for openapi_file in openapi_files: - app_path = openapi_file.parent.parent + for app_path in apps: manifest = get_manifest(app_path) if app_name and manifest.app_name != app_name: continue - if ClientType.PYTHON_CLIENT in client_types: - generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + if TemplateType.DJANGO_NINJA in manifest.templates: + generate_openapi_from_ninja_schema(manifest.app_name, app_path) + + for openapi_file in app_path.glob('api/*.yaml'): + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) - if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: - generate_ts_client(openapi_file) + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) aggregate_packages(client_src_path, client_lib_name) From 4b768ef829497428ee3930f37585f36346fa4e15 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Fri, 1 Nov 2024 18:27:25 +0000 Subject: [PATCH 10/15] CH-157 Move manifest management code into its own file --- .../ch_cli_tools/common_types.py | 16 +-- .../ch_cli_tools/manifest.py | 135 ++++++++++++++++++ tools/deployment-cli-tools/harness-generate | 87 +---------- 3 files changed, 140 insertions(+), 98 deletions(-) create mode 100644 tools/deployment-cli-tools/ch_cli_tools/manifest.py diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index 6c8e2c4c..dbff707a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -51,18 +51,4 @@ def to_dict(self) -> dict: 'version': self.version, 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], - } - - @classmethod - def migrate(cls, data: dict) -> tuple[dict, bool]: - data_copy = copy.deepcopy(data) - update_manifest = False - - if data_copy['version'] < '2': - update_manifest = True - data_copy['templates'] = [ - template if template != 'django-app' else 'django-fastapi' - for template in data_copy['templates'] - ] - - return data_copy, update_manifest \ No newline at end of file + } \ No newline at end of file diff --git a/tools/deployment-cli-tools/ch_cli_tools/manifest.py b/tools/deployment-cli-tools/ch_cli_tools/manifest.py new file mode 100644 index 00000000..155dc750 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/manifest.py @@ -0,0 +1,135 @@ +import abc +import copy +import logging +import pathlib +from typing import Iterable +from ruamel.yaml.error import YAMLError +from .common_types import CloudHarnessManifest, TemplateType +from .utils import load_yaml, save_yaml + + +def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: + manifest_file = app_path / '.ch-manifest' + + try: + manifest_data = load_yaml(manifest_file) + return CloudHarnessManifest.from_dict(manifest_data) + except (FileNotFoundError, YAMLError): + logging.info(f'Could not load manifest file {manifest_file}, inferring manifest from app structure...') + manifest = CloudHarnessManifest( + app_name=app_path.name, + inferred=True, + templates=infer_templates(app_path), + ) + save_yaml(manifest_file, manifest.to_dict()) + return manifest + + +def load_manifest(manifest_file: pathlib.Path) -> dict: + manifest_data = load_yaml(manifest_file) + migrated_data = migrate_manifest_data(manifest_data) + + if manifest_data != migrated_data: + save_yaml(manifest_file, migrated_data) + + return migrated_data + + +def migrate_manifest_data(data: dict) -> dict: + data = copy.deepcopy(data) + data_version = data['version'] + migrations = [ + migration for migration in _MIGRATIONS_LIST + if data_version < migration.change_version + ] + + for migration in migrations: + migration.migrate(data) + + return data + + +def infer_templates(app_path: pathlib.Path) -> list[str]: + return [ + TemplateType.BASE, + *infer_webapp_template(app_path), + *infer_server_template(app_path), + *infer_database_template(app_path), + ] + + +def infer_webapp_template(app_path: pathlib.Path) -> Iterable[str]: + frontend_path = app_path / 'frontend' + if frontend_path.exists(): + yield TemplateType.WEBAPP + + +def infer_server_template(app_path: pathlib.Path) -> Iterable[str]: + backend_path = app_path / 'backend' + manage_path = backend_path / 'manage.py' + + if manage_path.exists(): + yield from infer_django_template(backend_path) + return + + server_path = app_path / 'server' + if server_path.exists() or backend_path.exists(): + yield TemplateType.FLASK_SERVER + + +def infer_django_template(backend_path: pathlib.Path) -> Iterable[str]: + requirements_path = backend_path / 'requirements.txt' + requirements = requirements_path.read_text() + + if 'django-ninja' in requirements: + yield TemplateType.DJANGO_NINJA + else: + yield TemplateType.DJANGO_FASTAPI + + +def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: + values_file = app_path / 'deploy' / 'values.yaml' + + try: + values_data = load_yaml(values_file) + database_config = values_data['harness']['database'] + if not database_config['auto']: + return + + database_type = database_config['type'] + database_type_to_template_map = { + 'mongo': TemplateType.DB_MONGO, + 'neo4j': TemplateType.DB_NEO4J, + 'postgres': TemplateType.DB_POSTGRES, + } + + if database_type in database_type_to_template_map: + yield database_type_to_template_map[database_type] + + except(FileNotFoundError, YAMLError, KeyError): + pass + + +class ManifestMigration(abc.ABC): + @property + @abc.abstractmethod + def change_version(self) -> str: + ... + + @abc.abstractmethod + def migrate(data: dict) -> None: + ... + + +class NameChangeFromDjangoAppToDjangoFastapi(ManifestMigration): + change_version = '2' + + def migrate(data): + data['templates'] = [ + template if template != 'django-app' else 'django-fastapi' + for template in data['templates'] + ] + +_MIGRATIONS_LIST: list[ManifestMigration] = [ + NameChangeFromDjangoAppToDjangoFastapi(), +] \ No newline at end of file diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 545e6612..705a40c6 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -9,12 +9,12 @@ import pathlib import shutil import logging from typing import Callable, Optional -from ruamel.yaml.error import YAMLError from ch_cli_tools.openapi import LIB_NAME, generate_openapi_from_ninja_schema, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model -from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml -from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType +from ch_cli_tools.utils import copymergedir +from ch_cli_tools.common_types import TemplateType +from ch_cli_tools.manifest import get_manifest def main(): @@ -152,7 +152,7 @@ def generate_servers( """ Generates server stubs """ - openapi_files = get_openapi_file_paths(root_path) + openapi_files = [path for path in root_path.glob('applications/*/api/*.yaml')] for openapi_file in openapi_files: app_path = openapi_file.parent.parent @@ -207,10 +207,6 @@ def generate_clients( aggregate_packages(client_src_path, client_lib_name) -def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: - return [path for path in root_path.glob('applications/*/api/*.yaml')] - - def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): client_source_path.mkdir(parents=True, exist_ok=True) @@ -278,80 +274,5 @@ def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): shutil.rmtree(temp_module_path) -def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: - manifest_file = app_path / '.ch-manifest' - - try: - manifest_data = load_yaml(manifest_file) - manifest_data, update_manifest = CloudHarnessManifest.migrate(manifest_data) - if update_manifest: - save_yaml(manifest_file, manifest_data) - manifest = CloudHarnessManifest.from_dict(manifest_data) - except (FileNotFoundError, YAMLError): - logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') - manifest = CloudHarnessManifest( - app_name=app_path.name, - inferred=True, - templates=infer_templates(app_path), - ) - save_yaml(manifest_file, manifest.to_dict()) - - return manifest - - -def infer_templates(app_path: pathlib.Path) -> list[str]: - templates = [TemplateType.BASE] - - infer_webapp_template(app_path, templates) - infer_server_template(app_path, templates) - infer_database_template(app_path, templates) - - return templates - - -def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: - frontend_path = app_path / 'frontend' - if frontend_path.exists(): - templates.append(TemplateType.WEBAPP) - - -def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: - backend_path = app_path / 'backend' - manage_path = backend_path / 'manage.py' - - if manage_path.exists(): - requirements_path = backend_path / 'requirements.txt' - requirements = requirements_path.read_text() - if 'ninja' in requirements: - templates.append(TemplateType.DJANGO_NINJA) - else: - templates.append(TemplateType.DJANGO_FASTAPI) - return - - server_path = app_path / 'server' - if server_path.exists() or backend_path.exists(): - templates.append(TemplateType.FLASK_SERVER) - - -def infer_database_template(app_path: pathlib.Path, templates: list[str]) -> None: - values_file = app_path / 'deploy' / 'values.yaml' - - try: - values_data = load_yaml(values_file) - database_config = values_data['harness']['database'] - if not database_config['auto']: - return - - database_type = database_config['type'] - if database_type == 'mongo': - templates.append(TemplateType.DB_MONGO) - if database_type == 'neo4j': - templates.append(TemplateType.DB_NEO4J) - if database_type == 'postgres': - templates.append(TemplateType.DB_POSTGRES) - except (FileNotFoundError, YAMLError, KeyError): - pass - - if __name__ == "__main__": main() From 11590babcd48b844db282c32a0c1fbf2afa28e32 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 4 Nov 2024 17:20:39 +0000 Subject: [PATCH 11/15] CH-157 Fix linting issues --- .../ch_cli_tools/application_builders.py | 18 +++++++++--------- .../ch_cli_tools/common_types.py | 2 +- .../ch_cli_tools/manifest.py | 11 ++++++----- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py index ef020633..d7701445 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/application_builders.py +++ b/tools/deployment-cli-tools/ch_cli_tools/application_builders.py @@ -18,7 +18,7 @@ class ApplicationBuilder(abc.ABC): def __init__(self, app_name: str, app_path: pathlib.Path): self.app_name = app_name self.app_path = app_path - + @abc.abstractmethod def handles(self, templates: list[str]) -> bool: pass @@ -51,19 +51,19 @@ def merge_template_directories(self, template_name: str) -> None: @property def frontend_path(self): return self.app_path / 'frontend' - + @property def backend_path(self): return self.app_path / 'backend' - + @property def api_path(self): return self.app_path / 'api' - + @property def ch_root(self): return pathlib.Path(CH_ROOT) - + @property def app_template_path(self): return self.ch_root / APPLICATION_TEMPLATE_PATH @@ -103,7 +103,7 @@ def install_frontend_dependencies(self) -> None: class ServerAppBuilder(ApplicationBuilder): def handles(self, templates): return TemplateType.SERVER in templates - + def handle_pre_merge(self): with tempfile.TemporaryDirectory() as tmp_dirname: tmp_path = pathlib.Path(tmp_dirname) @@ -123,7 +123,7 @@ def handle_post_merge(self): class FlaskServerAppBuilder(ApplicationBuilder): def handles(self, templates): return TemplateType.FLASK_SERVER in templates - + def handle_pre_merge(self): pass @@ -149,7 +149,7 @@ def handle_post_merge(self): replace_in_file(self.app_path / 'dev-setup.sh', self.APP_NAME_PLACEHOLDER, self.app_name) self.create_django_app_vscode_debug_configuration() - + def create_django_app_vscode_debug_configuration(self): vscode_launch_path = pathlib.Path('.vscode/launch.json') configuration_name = f'{self.app_name} backend' @@ -173,7 +173,7 @@ def create_django_app_vscode_debug_configuration(self): @property def python_app_name(self): return to_python_module(self.app_name) - + @property @abc.abstractmethod def debug_template_file(self) -> str: diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py index dbff707a..eb1e5feb 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/common_types.py +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -51,4 +51,4 @@ def to_dict(self) -> dict: 'version': self.version, 'inferred': self.inferred, 'templates': [str(template) for template in self.templates], - } \ No newline at end of file + } diff --git a/tools/deployment-cli-tools/ch_cli_tools/manifest.py b/tools/deployment-cli-tools/ch_cli_tools/manifest.py index 155dc750..0425d806 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/manifest.py +++ b/tools/deployment-cli-tools/ch_cli_tools/manifest.py @@ -24,14 +24,14 @@ def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: save_yaml(manifest_file, manifest.to_dict()) return manifest - + def load_manifest(manifest_file: pathlib.Path) -> dict: manifest_data = load_yaml(manifest_file) migrated_data = migrate_manifest_data(manifest_data) if manifest_data != migrated_data: save_yaml(manifest_file, migrated_data) - + return migrated_data @@ -95,7 +95,7 @@ def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: database_config = values_data['harness']['database'] if not database_config['auto']: return - + database_type = database_config['type'] database_type_to_template_map = { 'mongo': TemplateType.DB_MONGO, @@ -106,7 +106,7 @@ def infer_database_template(app_path: pathlib.Path) -> Iterable[str]: if database_type in database_type_to_template_map: yield database_type_to_template_map[database_type] - except(FileNotFoundError, YAMLError, KeyError): + except (FileNotFoundError, YAMLError, KeyError): pass @@ -130,6 +130,7 @@ def migrate(data): for template in data['templates'] ] + _MIGRATIONS_LIST: list[ManifestMigration] = [ NameChangeFromDjangoAppToDjangoFastapi(), -] \ No newline at end of file +] From 82b44dad1888489854db7b6ec0e1c8d9de307330 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Tue, 12 Nov 2024 10:40:26 +0000 Subject: [PATCH 12/15] CH-157 Fix typo in django template readme files --- application-templates/django-fastapi/README.md | 2 +- application-templates/django-ninja/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-fastapi/README.md b/application-templates/django-fastapi/README.md index d77502b9..b237db70 100644 --- a/application-templates/django-fastapi/README.md +++ b/application-templates/django-fastapi/README.md @@ -31,7 +31,7 @@ See [backend/README.md#Develop] ### Frontend -Backend code is inside the *frontend* directory. +Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. diff --git a/application-templates/django-ninja/README.md b/application-templates/django-ninja/README.md index cc4b4c09..423842d6 100644 --- a/application-templates/django-ninja/README.md +++ b/application-templates/django-ninja/README.md @@ -30,7 +30,7 @@ See [backend/README.md#Develop] ### Frontend -Backend code is inside the *frontend* directory. +Frontend code is inside the *frontend* directory. Frontend is by default generated as a React web application, but no constraint about this specific technology. From 81f5dfb8fb38f4874c8e7003f907fbc7ef254c89 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:23:59 +0000 Subject: [PATCH 13/15] CH-157 Fixed typo in asgi and wsgi --- .../django-base/backend/django_baseapp/asgi.py | 4 ++-- .../django-base/backend/django_baseapp/wsgi.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application-templates/django-base/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py index 049df40e..76fc7817 100644 --- a/application-templates/django-base/backend/django_baseapp/asgi.py +++ b/application-templates/django-base/backend/django_baseapp/asgi.py @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() diff --git a/application-templates/django-base/backend/django_baseapp/wsgi.py b/application-templates/django-base/backend/django_baseapp/wsgi.py index e2d6fc09..fccfadef 100644 --- a/application-templates/django-base/backend/django_baseapp/wsgi.py +++ b/application-templates/django-base/backend/django_baseapp/wsgi.py @@ -21,6 +21,6 @@ init_services() # start the kafka event listener -from cloudharness_django.services.events import init_listner # noqa E402 +from cloudharness_django.services.events import init_listener # noqa E402 -init_listner() +init_listener() From a441b95d1d5c1aa8b0ef80eb8c12ea73225da287 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:47:18 +0000 Subject: [PATCH 14/15] CH-157 Fixed issue with incorrect django settings module for asgi --- .../django-base/backend/django_baseapp/asgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application-templates/django-base/backend/django_baseapp/asgi.py b/application-templates/django-base/backend/django_baseapp/asgi.py index 76fc7817..4ec66b7d 100644 --- a/application-templates/django-base/backend/django_baseapp/asgi.py +++ b/application-templates/django-base/backend/django_baseapp/asgi.py @@ -11,7 +11,7 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "__APP_NAME__.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_baseapp.settings") application = get_asgi_application() From 23a6fed7514118c966771b44b13318bc6bbfa151 Mon Sep 17 00:00:00 2001 From: Jake Conkerton-Darby Date: Mon, 9 Dec 2024 11:47:48 +0000 Subject: [PATCH 15/15] CH-157 Change name of frontend build stage to be more generic --- application-templates/django-ninja/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application-templates/django-ninja/Dockerfile b/application-templates/django-ninja/Dockerfile index f13bfaac..dc8e2223 100644 --- a/application-templates/django-ninja/Dockerfile +++ b/application-templates/django-ninja/Dockerfile @@ -1,7 +1,7 @@ -ARG MNP_UI +ARG CLOUDHARNESS_FRONTEND_BUILD ARG CLOUDHARNESS_DJANGO -FROM $MNP_UI AS frontend +FROM $CLOUDHARNESS_FRONTEND_BUILD AS frontend ARG APP_DIR=/app