From f61aa7f4aab68a77817d840cbe021ccc79385573 Mon Sep 17 00:00:00 2001 From: Nathan Freeman Date: Thu, 11 Jul 2024 15:10:01 -0500 Subject: [PATCH] fix patch task --- src/api/src/backend/models.py | 6 +- .../backend/serializers/BaseTaskSerializer.py | 27 +++ .../serializers/FunctionTaskSerializer.py | 16 +- .../src/backend/serializers/TaskSerializer.py | 14 +- src/api/src/backend/serializers/__init__.py | 1 + src/api/src/backend/views/Tasks.py | 9 +- src/api/src/tests/TestModelSerializers.py | 24 +++ src/api/src/tests/run.sh | 9 +- src/api/src/workflows/settings.py | 14 +- src/api/src/workflows/settings_test.py | 158 ++++++++++++++++++ 10 files changed, 255 insertions(+), 23 deletions(-) create mode 100644 src/api/src/backend/serializers/BaseTaskSerializer.py create mode 100644 src/api/src/tests/TestModelSerializers.py create mode 100644 src/api/src/workflows/settings_test.py diff --git a/src/api/src/backend/models.py b/src/api/src/backend/models.py index 0024e4a1..9cf89c68 100644 --- a/src/api/src/backend/models.py +++ b/src/api/src/backend/models.py @@ -450,10 +450,10 @@ class Meta: # Props id = models.CharField(validators=[validate_id], max_length=128) cache = models.BooleanField(null=True) + conditions = models.JSONField(null=True, default=list) depends_on = models.JSONField(null=True, default=list) description = models.TextField(null=True) flavor = models.CharField(max_length=32, choices=TASK_FLAVORS, default=TASK_FLAVOR_C1_MED) - conditions = models.JSONField(null=True, default=list) input = models.JSONField(null=True) invocation_mode = models.CharField(max_length=16, default=EnumInvocationMode.Async) max_exec_time = models.BigIntegerField( @@ -463,7 +463,6 @@ class Meta: max_retries = models.IntegerField(default=DEFAULT_MAX_RETRIES) output = models.JSONField(null=True) pipeline = models.ForeignKey("backend.Pipeline", related_name="tasks", on_delete=models.CASCADE) - poll = models.BooleanField(null=True) retry_policy = models.CharField(max_length=32, default=EnumRetryPolicy.ExponentialBackoff) type = models.CharField(max_length=32, choices=TASK_TYPES) uses = models.JSONField(null=True) @@ -503,6 +502,9 @@ class Meta: tapis_actor_id = models.CharField(max_length=128, null=True) tapis_actor_message = models.TextField(null=True) + # Shared properties (Tapis job and Tapis actor) + poll = models.BooleanField(null=True) + def clean(self): errors = {} diff --git a/src/api/src/backend/serializers/BaseTaskSerializer.py b/src/api/src/backend/serializers/BaseTaskSerializer.py new file mode 100644 index 00000000..d9f75278 --- /dev/null +++ b/src/api/src/backend/serializers/BaseTaskSerializer.py @@ -0,0 +1,27 @@ +from backend.serializers import UUIDSerializer + +class BaseTaskSerializer: + @staticmethod + def serialize(model): + task = {} + task["id"] = model.id + task["cache"] = model.cache + task["conditions"] = model.conditions + task["depends_on"] = model.depends_on + task["description"] = model.description + task["flavor"] = model.flavor + task["input"] = model.input + task["output"] = model.output + task["type"] = model.type + task["uses"] = model.uses + task["uuid"] = UUIDSerializer.serialize(model.uuid) + + # Build execution profile + task["execution_profile"] = { + "invocation_mode": model.invocation_mode, + "max_exec_time": model.max_exec_time, + "max_retries": model.max_retries, + "retry_policy": model.retry_policy, + } + + return task \ No newline at end of file diff --git a/src/api/src/backend/serializers/FunctionTaskSerializer.py b/src/api/src/backend/serializers/FunctionTaskSerializer.py index 745e978b..5d0ceb9a 100644 --- a/src/api/src/backend/serializers/FunctionTaskSerializer.py +++ b/src/api/src/backend/serializers/FunctionTaskSerializer.py @@ -1,6 +1,16 @@ -from backend.serializers import FunctionTaskSerializer +from backend.serializers import BaseTaskSerializer class FunctionTaskSerializer: @staticmethod - def serialize(model): - return \ No newline at end of file + def serialize(model, base=None): + task = base if base != None else BaseTaskSerializer.serialize(model) + + task["git_repositories"] = model.git_repositories + task["code"] = model.code + task["runtime"] = model.runtime + task["command"] = model.command + task["installer"] = model.installer + task["packages"] = model.packages + task["entrypoint"] = model.entrypoint + + return task \ No newline at end of file diff --git a/src/api/src/backend/serializers/TaskSerializer.py b/src/api/src/backend/serializers/TaskSerializer.py index 4be99d4c..8fffb35e 100644 --- a/src/api/src/backend/serializers/TaskSerializer.py +++ b/src/api/src/backend/serializers/TaskSerializer.py @@ -1,7 +1,15 @@ -from backend.serializers import FunctionTaskSerializer +from backend.serializers import ( + FunctionTaskSerializer, + BaseTaskSerializer, +) +from backend.models import TASK_TYPE_FUNCTION class TaskSerializer: @staticmethod def serialize(model): - if model.type == "function": - return FunctionTaskSerializer.serialize(model) \ No newline at end of file + base = BaseTaskSerializer(model) + + if model.type == TASK_TYPE_FUNCTION: + return FunctionTaskSerializer.serialize(model, base=base) + + raise NotImplementedError(f"Task Serializer does not have a method for serializing tasks of type '{model.type}'") \ No newline at end of file diff --git a/src/api/src/backend/serializers/__init__.py b/src/api/src/backend/serializers/__init__.py index 2109c20f..e35a6630 100644 --- a/src/api/src/backend/serializers/__init__.py +++ b/src/api/src/backend/serializers/__init__.py @@ -1,5 +1,6 @@ from backend.serializers.UUIDSerializer import UUIDSerializer from backend.serializers.PipelineLockModelSerializer import PipelineLockModelSerializer from backend.serializers.PipelineLockAcquisitionResponseSerializer import PipelineLockAcquisitionResponseSerializer +from backend.serializers.BaseTaskSerializer import BaseTaskSerializer from backend.serializers.TaskSerializer import TaskSerializer from backend.serializers.FunctionTaskSerializer import FunctionTaskSerializer \ No newline at end of file diff --git a/src/api/src/backend/views/Tasks.py b/src/api/src/backend/views/Tasks.py index c6daf6ce..ef948791 100644 --- a/src/api/src/backend/views/Tasks.py +++ b/src/api/src/backend/views/Tasks.py @@ -1,7 +1,6 @@ import json, pprint from django.db import IntegrityError, OperationalError, DatabaseError -from django.forms import model_to_dict from backend.models import Task, Pipeline from backend.views.RestrictedAPIView import RestrictedAPIView @@ -10,6 +9,7 @@ from backend.views.http.responses.models import ModelListResponse, ModelResponse from backend.services.TaskService import service as task_service from backend.services.GroupService import service as group_service +from backend.serializers import TaskSerializer from backend.errors.api import ServerError as APIServerError from backend.helpers import resource_url_builder from backend.utils import logger @@ -131,13 +131,8 @@ def patch(self, request, group_id, pipeline_id, task_id): # Resolve the the proper pydantic object for this task type TaskSchema = task_service.resolve_request_type(task_model.type) - serialized_task = model_to_dict(task_model) - - print(serialized_task) - pprint(serialized_task) - task = TaskSchema(**{ - **serialized_task, + **TaskSerializer.serialize(task_model), **self.request_body }) diff --git a/src/api/src/tests/TestModelSerializers.py b/src/api/src/tests/TestModelSerializers.py new file mode 100644 index 00000000..52bdfc31 --- /dev/null +++ b/src/api/src/tests/TestModelSerializers.py @@ -0,0 +1,24 @@ +import unittest +from django.test import TestCase + +# from backend.models import ( +# Task +# ) +# from backend.serializers import ( +# TaskSerializer +# ) + +# from tests.utils import load_fixture + + +class TestModelSerializers(TestCase): + def testTask(self): + pass + # mock = MagicMock(spec=Task) + # data = load_fixture("function-task.json") + # for k, v in data.items(): + # setattr(mock, k, v) + + # task = TaskSerializer.serialize(mock) + + diff --git a/src/api/src/tests/run.sh b/src/api/src/tests/run.sh index f6576a27..7b777c21 100755 --- a/src/api/src/tests/run.sh +++ b/src/api/src/tests/run.sh @@ -1,6 +1,13 @@ cd $(dirname $0) cd ../ +export DJANGO_SETTINGS_MODULE=workflows.settings_test +export ENV=LOCAL + python3 -m unittest -v tests.TestUrls python3 -m unittest -v tests.TestImports -python3 -m unittest -v tests.TestTapisETLPipeline \ No newline at end of file +python3 -m unittest -v tests.TestTapisETLPipeline + +# Tests that require django models must be tested through the testing utilities +# provided by django +# python3 manage.py test tests.TestModelSerializers \ No newline at end of file diff --git a/src/api/src/workflows/settings.py b/src/api/src/workflows/settings.py index bea4293c..60276c4d 100644 --- a/src/api/src/workflows/settings.py +++ b/src/api/src/workflows/settings.py @@ -19,7 +19,7 @@ ENVS = ["LOCAL", "DEV", "STAGE", "PROD"] # The environment in which the application is currently deployed -ENV = os.environ["ENV"] +ENV = os.environ.get("ENV") # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -28,10 +28,10 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -LOG_LEVEL = os.environ["LOG_LEVEL"] +LOG_LEVEL = os.environ.get("LOG_LEVEL") DEBUG = True if ENV != "PROD" else False # LOGGING = { @@ -139,10 +139,10 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.mysql", - "NAME": os.environ["DB_NAME"], - "HOST": os.environ["DB_HOST"], - "USER": os.environ["DB_USER"], - "PASSWORD": os.environ["DB_PASSWORD"] + "NAME": os.environ.get("DB_NAME"), + "HOST": os.environ.get("DB_HOST"), + "USER": os.environ.get("DB_USER"), + "PASSWORD": os.environ.get("DB_PASSWORD") } } diff --git a/src/api/src/workflows/settings_test.py b/src/api/src/workflows/settings_test.py new file mode 100644 index 00000000..08707cf5 --- /dev/null +++ b/src/api/src/workflows/settings_test.py @@ -0,0 +1,158 @@ +import os + +from pathlib import Path + +# List of all acceptable ENV values +ENVS = ["LOCAL", "DEV", "STAGE", "PROD"] + +# The environment in which the application is currently deployed +ENV = os.environ.get("ENV") + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "abcdefg" + +# SECURITY WARNING: don't run with debug turned on in production! +LOG_LEVEL = os.environ.get("LOG_LEVEL") +DEBUG = True if ENV != "PROD" else False + +# LOGGING = { +# 'version': 1, +# 'disable_existing_loggers': True, +# 'handlers': { +# 'file': { +# 'level': LOG_LEVEL, +# 'class': 'logging.FileHandler', +# 'filename': f'./logs/{LOG_LEVEL}.logs', +# }, +# }, +# 'loggers': { +# 'django': { +# 'handlers': ['file'], +# 'level': LOG_LEVEL, +# 'propagate': True, +# }, +# }, +# } + +# Set allowed hosts by env +ALLOWED_HOSTS = ['*'] + + +# Application definition +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.messages', + 'django.contrib.sites', + 'django.contrib.staticfiles', + 'rest_framework', + # 'corsheaders', + 'backend', +] + +MIDDLEWARE = [ + # 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +# TODO make more restrictive by implementing the CORS_ORIGIN_WHITELIST +# CORS_ALLOW_ALL_ORIGINS = True + +CORS_ORIGIN_WHITELIST = () +# that are not located at .tapis.io to run this api +if ENV == "LOCAL": + CORS_ORIGIN_WHITELIST = ("localhost", "127.0.0.1") +elif ENV == "DEV": + CORS_ORIGIN_WHITELIST = ("localhost", "127.0.0.1") +elif ENV == "STAGE": + CORS_ORIGIN_WHITELIST = ("localhost", "127.0.0.1") +elif ENV == "PROD": + CORS_ORIGIN_WHITELIST = ("localhost", "127.0.0.1") +else: + raise Exception(f"Invalid ENV set. Recieved '{ENV}' Expected oneOf: {ENVS}") + +ROOT_URLCONF = 'workflows.urls' + +# Set url prefix based on env +URL_PREFIX = "" if ENV == "LOCAL" else "v3/workflows/" + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'workflows.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ +STATIC_URL = '/static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' \ No newline at end of file